openc3 7.4.0 → 7.5.0

data/lib/openc3/packets/packet_item.rb

@@ -85,8 +85,22 @@ module OpenC3
     # The allowable state colors
     STATE_COLORS = [:GREEN, :YELLOW, :RED]
 
+    # PacketItemLimits is created on first access since most items have no
+    # limits and large definitions have hundreds of thousands of items.
     # @return [PacketItemLimits] All information regarding limits for this PacketItem
-    attr_reader :limits
+    def limits
+      @limits ||= PacketItemLimits.new
+    end
+
+    # @return [Symbol, nil] The limits state without creating PacketItemLimits
+    def limits_state
+      @limits ? @limits.state : nil
+    end
+
+    # @return [Hash, nil] The limits values without creating PacketItemLimits
+    def limits_values
+      @limits ? @limits.values : nil
+    end
 
     # @return [Boolean] Whether the parameter must be obfuscated from logs or not
     attr_accessor :obfuscate
@@ -110,7 +124,7 @@ module OpenC3
       @obfuscate = false
       @messages_disabled = nil
       @state_colors = nil
-      @limits = PacketItemLimits.new
+      @limits = nil
       @meta = nil
     end
 
@@ -165,7 +179,6 @@ module OpenC3
         end
 
         @states = upcase_states
-        @state_colors ||= {}
       else
         @states = nil
       end
@@ -317,7 +330,7 @@ module OpenC3
       item.hazardous = self.hazardous.clone if self.hazardous
       item.messages_disabled = self.messages_disabled.clone if self.messages_disabled
       item.state_colors = self.state_colors.clone if self.state_colors
-      item.limits = self.limits.clone if self.limits
+      item.limits = @limits.clone if @limits
       item.meta = self.meta.clone if @meta
       item.obfuscate = self.obfuscate.clone if @obfuscate
       item
@@ -401,16 +414,16 @@ module OpenC3
       config << self.read_conversion.to_config(:READ) if self.read_conversion
       config << self.write_conversion.to_config(:WRITE) if self.write_conversion
 
-      if self.limits.values
-        self.limits.values.each do |limits_set, limits_values|
-          config << "    LIMITS #{limits_set} #{self.limits.persistence_setting} #{self.limits.enabled ? 'ENABLED' : 'DISABLED'} #{limits_values[0]} #{limits_values[1]} #{limits_values[2]} #{limits_values[3]}"
+      if limits_values
+        @limits.values.each do |limits_set, limits_values|
+          config << "    LIMITS #{limits_set} #{@limits.persistence_setting} #{@limits.enabled ? 'ENABLED' : 'DISABLED'} #{limits_values[0]} #{limits_values[1]} #{limits_values[2]} #{limits_values[3]}"
           if limits_values[4] && limits_values[5]
             config << " #{limits_values[4]} #{limits_values[5]}\n"
           else
             config << "\n"
           end
         end
-        config << self.limits.response.to_config if self.limits.response
+        config << @limits.response.to_config if @limits.response
       end
 
       if @meta
@@ -456,19 +469,19 @@ module OpenC3
       config['write_conversion'] = self.write_conversion.as_json(*a) if self.write_conversion
 
       config['limits'] ||= {}
-      if self.limits.enabled
+      if @limits && @limits.enabled
         config['limits']['enabled'] = true
-      elsif self.limits.values || (self.state_colors && self.state_colors.length > 0)
+      elsif limits_values || (self.state_colors && self.state_colors.length > 0)
         # Only set to false if there are limits or state colors
         # to avoid items without limits acting like they can be enabled
         config['limits']['enabled'] = false
       end
-      if self.limits.values
+      if limits_values
         # Only set these if there are limits.values because persistence_setting has a default
         # and we don't want keys on the 'limits' hash if there aren't any limits
-        config['limits']['persistence_setting'] = self.limits.persistence_setting if self.limits.persistence_setting
-        config['limits']['response'] = self.limits.response.to_s if self.limits.response
-        self.limits.values.each do |limits_set, limits_values|
+        config['limits']['persistence_setting'] = @limits.persistence_setting if @limits.persistence_setting
+        config['limits']['response'] = @limits.response.to_s if @limits.response
+        @limits.values.each do |limits_set, limits_values|
           limits = {}
           limits['red_low'] =  limits_values[0]
           limits['yellow_low'] = limits_values[1]

data/lib/openc3/packets/parsers/state_parser.rb

@@ -26,7 +26,7 @@ module OpenC3
     # @param warnings [Array<String>] Array of string warnings which will be
     #   appended with any warnings found when parsing the limits
     def self.parse(parser, packet, cmd_or_tlm, item, warnings)
-      raise parser.error("Items with LIMITS can't define STATE") if item.limits.values
+      raise parser.error("Items with LIMITS can't define STATE") if item.limits_values
       raise parser.error("Items with UNITS can't define STATE") if item.units
 
       parser = StateParser.new(parser)

data/lib/openc3/packets/parsers/xtce_parser.rb

@@ -22,6 +22,10 @@ module OpenC3
   class XtceParser
     attr_accessor :current_target_name
 
+    # The target the file defined, still readable after parsing finishes.
+    # current_target_name is cleared by reset_processing_variables.
+    attr_reader :parsed_target_name
+
     # Processes a XTCE formatted OpenC3 configuration file
     #
     # @param commands [Hash<String=>Packet>] Hash of all the command packets
@@ -32,8 +36,10 @@ module OpenC3
     #   that were created while parsing the configuration
     # @param filename [String] The name of the configuration file
     # @param target_name [String] Override the target name found in the XTCE file
+    # @return [String] The target name the packets were added under. The caller
+    #   needs this to build the ID lookup metadata for those packets.
     def self.process(commands, telemetry, warnings, filename, target_name = nil)
-      XtceParser.new(commands, telemetry, warnings, filename, target_name)
+      XtceParser.new(commands, telemetry, warnings, filename, target_name).parsed_target_name
     end
 
     def self.reverse_packet_order(target_name, cmd_or_tlm_hash)
@@ -91,6 +97,7 @@ module OpenC3
       XtceParser.reverse_packet_order(@current_target_name, @commands)
       XtceParser.reverse_packet_order(@current_target_name, @telemetry)
 
+      @parsed_target_name = @current_target_name
       reset_processing_variables()
     end
 

data/lib/openc3/packets/structure_item.rb

@@ -143,7 +143,8 @@ module OpenC3
       raise ArgumentError, "name must be a String but is a #{name.class}" unless String === name
       raise ArgumentError, "name must contain at least one character" if name.empty?
 
-      @name = name.upcase.clone.freeze
+      # Dedup since many packets share the same item names
+      @name = -name.upcase
       verify_overall() if @structure_item_constructed
     end
 
@@ -151,7 +152,7 @@ module OpenC3
       raise ArgumentError, "key must be a String but is a #{key.class}" unless String === key
       raise ArgumentError, "key must contain at least one character" if key.empty?
 
-      @key = key
+      @key = -key
     end
 
     def endianness=(endianness)

data/lib/openc3/script/api_shared.rb

@@ -530,7 +530,7 @@ module OpenC3
 
       value = yield(target_name, packet_name, item_name)
       if comparison_to_eval
-        _check_eval(target_name, packet_name, item_name, comparison_to_eval, value)
+        _check_comparison(target_name, packet_name, item_name, comparison_to_eval, value)
       else
         puts "CHECK: #{_upcase(target_name, packet_name, item_name)} == #{value.nil? ? 'nil' : value.inspect}"
       end
@@ -733,28 +733,16 @@ module OpenC3
       return [target_name, packet_name, item_name, comparison_to_eval, timeout, polling_rate]
     end
 
-    def _openc3_script_wait_implementation(target_name, packet_name, item_name, value_type, timeout, polling_rate, exp_to_eval, scope: $openc3_scope, token: $openc3_token, &block)
+    # Waits for the comparison to be true or the timeout to expire.
+    # The comparison is a callable which takes the telemetry value and returns true or false.
+    # A block passed by the user takes precedence over the comparison.
+    def _openc3_script_wait_implementation(target_name, packet_name, item_name, value_type, timeout, polling_rate, comparison, scope: $openc3_scope, token: $openc3_token, &block)
+      condition = block || comparison
       end_time = Time.now.sys + timeout
-      if exp_to_eval and !exp_to_eval.is_printable?
-        raise "ERROR: Invalid comparison to non-ascii value"
-      end
       while true
         work_start = Time.now.sys
         value = tlm(target_name, packet_name, item_name, type: value_type, scope: scope, token: token)
-        if not block.nil?
-          if block.call(value)
-            return true, value
-          end
-        else
-          begin
-            if eval(exp_to_eval)
-              return true, value
-            end
-          # NoMethodError is raised when the tlm() returns nil and we try to eval the expression
-          # In this case we just continue and see if eventually we get a good value from tlm()
-          rescue NoMethodError
-          end
-        end
+        return true, value if condition and condition.call(value)
         break if Time.now.sys >= end_time
 
         delta = Time.now.sys - work_start
@@ -766,58 +754,62 @@ module OpenC3
 
         if canceled
           value = tlm(target_name, packet_name, item_name, type: value_type, scope: scope, token: token)
-          if not block.nil?
-            if block.call(value)
-              return true, value
-            else
-              return false, value
-            end
+          if condition and condition.call(value)
+            return true, value
           else
-            begin
-              if eval(exp_to_eval)
-                return true, value
-              else
-                return false, value
-              end
-            # NoMethodError is raised when the tlm() returns nil and we try to eval the expression
-            rescue NoMethodError
-              return false, value
-            end
+            return false, value
           end
         end
       end
 
       return false, value
-    rescue NameError => e
-      if e.message =~ /uninitialized constant OpenC3::ApiShared::(\w+)/
-        new_error = NameError.new("Uninitialized constant #{$1}. Did you mean '#{$1}' as a string?")
-        new_error.set_backtrace(e.backtrace)
-        raise new_error
-      else
-        raise e
-      end
+    end
+
+    # Builds a callable which compares a telemetry value against the given comparison string,
+    # e.g. "> 1". Returns nil if there is no comparison, e.g. a block based wait_check().
+    # Raises if the comparison is invalid, e.g. an unsupported operator or unparsable operand.
+    def _comparison_implementation(comparison_to_eval)
+      return nil unless comparison_to_eval
+
+      operator, operand = extract_operator_and_operand_from_comparison(comparison_to_eval)
+      return nil unless operator
+      lambda { |value| compare_values(value, operator, operand) }
     end
 
     # Wait for a converted telemetry item to pass a comparison
     def _openc3_script_wait_implementation_comparison(target_name, packet_name, item_name, value_type, comparison_to_eval, timeout, polling_rate = DEFAULT_TLM_POLLING_RATE, scope: $openc3_scope, token: $openc3_token, &block)
-      if comparison_to_eval
-        exp_to_eval = "value " + comparison_to_eval
-      else
-        exp_to_eval = nil
+      # The comparison text is logged whether or not it is parsed so it is always validated
+      if comparison_to_eval and !comparison_to_eval.is_printable?
+        raise "ERROR: Invalid comparison to non-ascii value"
       end
-      _openc3_script_wait_implementation(target_name, packet_name, item_name, value_type, timeout, polling_rate, exp_to_eval, scope: scope, token: token, &block)
+      # A user supplied block is the condition so the comparison text is never parsed
+      comparison = block ? nil : _comparison_implementation(comparison_to_eval)
+      _openc3_script_wait_implementation(target_name, packet_name, item_name, value_type, timeout, polling_rate, comparison, scope: scope, token: token, &block)
     end
 
     def _openc3_script_wait_implementation_tolerance(target_name, packet_name, item_name, value_type, expected_value, tolerance, timeout, polling_rate = DEFAULT_TLM_POLLING_RATE, scope: $openc3_scope, token: $openc3_token, &block)
-      exp_to_eval = "((#{expected_value} - #{tolerance})..(#{expected_value} + #{tolerance})).include? value"
-      _openc3_script_wait_implementation(target_name, packet_name, item_name, value_type, timeout, polling_rate, exp_to_eval, scope: scope, token: token, &block)
+      comparison = lambda do |value|
+        begin
+          value >= (expected_value - tolerance) and value <= (expected_value + tolerance)
+        rescue ArgumentError, NoMethodError, TypeError
+          false
+        end
+      end
+      _openc3_script_wait_implementation(target_name, packet_name, item_name, value_type, timeout, polling_rate, comparison, scope: scope, token: token, &block)
     end
 
     def _openc3_script_wait_implementation_array_tolerance(array_size, target_name, packet_name, item_name, value_type, expected_value, tolerance, timeout, polling_rate = DEFAULT_TLM_POLLING_RATE, scope: $openc3_scope, token: $openc3_token, &block)
-      statements = []
-      array_size.times { |i| statements << "(((#{expected_value[i]} - #{tolerance[i]})..(#{expected_value[i]} + #{tolerance[i]})).include? value[#{i}])" }
-      exp_to_eval = statements.join(" && ")
-      _openc3_script_wait_implementation(target_name, packet_name, item_name, value_type, timeout, polling_rate, exp_to_eval, scope: scope, token: token, &block)
+      comparison = lambda do |values|
+        begin
+          next false unless values.is_a?(Array) and values.length == array_size
+          array_size.times.all? do |i|
+            values[i] >= (expected_value[i] - tolerance[i]) and values[i] <= (expected_value[i] + tolerance[i])
+          end
+        rescue ArgumentError, NoMethodError, TypeError
+          false
+        end
+      end
+      _openc3_script_wait_implementation(target_name, packet_name, item_name, value_type, timeout, polling_rate, comparison, scope: scope, token: token, &block)
     end
 
     # Wait on an expression to be true.
@@ -859,17 +851,15 @@ module OpenC3
       end
     end
 
-    def _check_eval(target_name, packet_name, item_name, comparison_to_eval, value)
-      string = "value " + comparison_to_eval
+    def _check_comparison(target_name, packet_name, item_name, comparison_to_eval, value)
       check_str = "CHECK: #{_upcase(target_name, packet_name, item_name)} #{comparison_to_eval}"
       # Show user the check against a quoted string
-      # Note: We have to preserve the original 'value' variable because we're going to eval against it
       value_str = value.is_a?(String) ? "'#{value}'" : value
       value_str = 'nil' if value.nil? # Show user nil value as 'nil'
       with_value = "with value == #{value_str}"
 
-      eval_is_valid = _check_eval_validity(value, comparison_to_eval)
-      unless eval_is_valid
+      operator, operand = extract_operator_and_operand_from_comparison(comparison_to_eval)
+      unless _valid_comparison?(value, operator, operand)
         message = "Invalid comparison for types"
         if $disconnect
           puts "ERROR: #{message}"
@@ -878,7 +868,7 @@ module OpenC3
         end
       end
 
-      if eval_is_valid && eval(string)
+      if compare_values(value, operator, operand)
         puts "#{check_str} success #{with_value}"
       else
         message = "#{check_str} failed #{with_value}"
@@ -888,35 +878,19 @@ module OpenC3
           raise CheckError, message
         end
       end
-    rescue NameError => e
-      if e.message =~ /uninitialized constant OpenC3::ApiShared::(\w+)/
-        new_error = NameError.new("Uninitialized constant #{$1}. Did you mean '#{$1}' as a string?")
-        new_error.set_backtrace(e.backtrace)
-        raise new_error
-      else
-        raise e
-      end
     end
 
-    def _check_eval_validity(value, comparison)
-      return true if comparison.nil? || comparison.empty?
-
-      begin
-        operator, operand = extract_operator_and_operand_from_comparison(comparison)
-      rescue RuntimeError => e
-        if e.message.include?("Unable to parse operand")
-          # If we can't parse the operand, let the eval happen anyway
-          # It will raise an appropriate error (like NameError for undefined constants)
-          return true
-        end
-        raise # Re-raise invalid operator errors
-      rescue JSON::ParserError
-        return true
-      end
+    # Returns whether the value and operand can be meaningfully compared with the operator.
+    # Note this is only used by check() because wait() polls until the value changes.
+    def _valid_comparison?(value, operator, operand)
+      return true if operator.nil?
 
       if [">=", "<=", ">", "<"].include?(operator)
-        return false if value.nil? || operand.nil? || value.is_a?(Array) || operand.is_a?(Array)
+        return false if value.nil? or operand.nil?
+        return false if value.is_a?(Array) or operand.is_a?(Array)
+        return false if value.is_a?(String) != operand.is_a?(String)
       end
+      # Note 'in' does not need a check here because the parser already requires a list operand
 
       return true
     end

data/lib/openc3/script/extract.rb

@@ -26,6 +26,37 @@ module OpenC3
     # whitespace around them is optional.
     SCANNING_REGULAR_EXPRESSION = %r{ (?:"(?:[^\\"]|\\.)*") | (?:'(?:[^\\']|\\.)*') | (?:\[(?:[^\\\[\]]|\\.|\[(?:[^\\\[\]]|\\.)*\])*\]) | , | [^\s,]+ }x # "
 
+    # Operators supported by check(), wait() and wait_check() comparisons
+    COMPARISON_OPERATORS = ["==", "!=", ">=", "<=", ">", "<", "in"]
+
+    # Single character escape sequences processed inside a double quoted comparison operand.
+    # The numeric (\nnn, \xHH, \uHHHH, \u{...}) forms are handled by unescape_double_quoted().
+    DOUBLE_QUOTE_ESCAPES = {
+      '\\' => "\\", '"' => '"', "'" => "'", '#' => '#', 'n' => "\n", 't' => "\t",
+      'r' => "\r", 'a' => "\a", 'b' => "\b", 'e' => "\e", 'f' => "\f", 'v' => "\v", 's' => ' '
+    }
+
+    # Tokenizes a double quoted operand into escape sequences, interpolation markers and runs
+    # of plain characters. The alternatives are ordered longest first so \u{1F600} is not read
+    # as \u, and the escape alternatives precede the markers so \#{ is a literal '#{'.
+    DOUBLE_QUOTE_TOKEN_REGEX =
+      /\\u\{[\h\s]*\}|\\u\h{4}|\\x\h{1,2}|\\[0-7]{1,3}|\\M-\\C-.|\\M-.|\\C-.|\\c.|\\.|\#[{@$]|[^\\\#]+|\#/m
+
+    # Escape sequences which have a meaning we deliberately do not implement. Dropping the
+    # backslash would silently change the value so they are rejected instead.
+    UNSUPPORTED_ESCAPE_REGEX = /\A\\(?:c|C-|M-)/
+
+    # The Ruby string interpolation markers. Interpolation would be code execution so it is
+    # rejected rather than silently compared against the uninterpolated text.
+    INTERPOLATION_REGEX = /\A\#[{@$]/
+
+    # Limits Ruby enforces on a \u escape. A codepoint is written with at most six hex digits,
+    # the largest character Unicode defines is 0x10FFFF, and the surrogate range is reserved
+    # for UTF-16 pairs so it is not a character on its own.
+    MAX_UNICODE_DIGITS = 6
+    MAX_UNICODE_CODEPOINT = 0x10FFFF
+    UNICODE_SURROGATE_RANGE = (0xD800..0xDFFF)
+
     private
 
     # Pulls all string keyword arguments into the args array. Raises on any symbol keyword arguments.
@@ -165,8 +196,6 @@ module OpenC3
 
     # Splits `check()` comparison expressions, e.g. "== 'foo bar'" becomes ["==", "foo bar"]
     def extract_operator_and_operand_from_comparison(comparison)
-      valid_operators = ["==", "!=", ">=", "<=", ">", "<", "in"]
-
       operator, operand = comparison.split(nil, 2) # Ruby: second split arg is max number of resultant elements
 
       if operand.nil?
@@ -175,29 +204,196 @@ module OpenC3
         return [nil, nil]
       end
 
-      raise "ERROR: Invalid operator: '#{operator}'" unless valid_operators.include?(operator)
+      raise "ERROR: Invalid operator: '#{operator}'" unless COMPARISON_OPERATORS.include?(operator)
+
+      # split leaves any trailing whitespace on the operand which the anchored matches reject
+      operand = extract_operand(operand.strip)
+      # 'in' is containment against a list of values in both Ruby and Python.
+      # Enforced here so check(), wait() and wait_check() all reject the same thing.
+      if operator == "in" and !operand.is_a?(Array)
+        raise "ERROR: The 'in' operator requires a list operand: #{operand.inspect}"
+      end
+
+      return [operator, operand]
+    end
 
-      # Handle string operand: remove surrounding double/single quotes
-      if operand.match?(/^(['"])(.*)\1$/m) # Starts with single or double quote, and ends with matching quote
-        operand = operand[1..-2]
-        return [operator, operand]
+    # Converts the operand of a `check()` comparison expression into a Ruby value.
+    # Note this deliberately does not eval the operand so only literal values are supported.
+    def extract_operand(operand)
+      # A quoted operand must be a single complete string literal. Anything trailing the
+      # closing quote, e.g. "== 'a' garbage 'b'", is a syntax error rather than a string.
+      if (match = operand.match(/\A'((?:[^'\\]|\\.)*)'\z/m))
+        # Ruby single quoted strings only escape the backslash and the single quote
+        return match[1].gsub(/\\([\\'])/) { $1 }
+      end
+      if (match = operand.match(/\A"((?:[^"\\]|\\.)*)"\z/m))
+        return unescape_double_quoted(match[1])
       end
+      return nil if operand == "nil"
+      return false if operand == "false"
+      return true if operand == "true"
 
-      # Handle other operand types
-      if operand == "nil"
-        operand = nil
-      elsif operand == "false"
-        operand = false
-      elsif operand == "true"
-        operand = true
+      # Ruby's Float() rejects these but Python's float() accepts them
+      case operand.upcase
+      when 'INFINITY', '+INFINITY', 'INF', '+INF'
+        return Float::INFINITY
+      when '-INFINITY', '-INF'
+        return -Float::INFINITY
+      when 'NAN', '+NAN', '-NAN'
+        return Float::NAN
       else
-        begin
-          operand = JSON.parse(operand)
-        rescue JSON::ParserError
-          raise "ERROR: Unable to parse operand: #{operand}"
+        # Not an infinity or NaN keyword so fall through to the formats below
+      end
+
+      # Arrays are parsed recursively so their elements follow exactly the same rules as a
+      # bare operand. JSON alone would reject ['ON', 'OFF'], [0xA, 0xB] and [true, nil].
+      if operand.start_with?('[') and operand.end_with?(']')
+        return split_operand_list(operand[1..-2]).map { |element| extract_operand(element) }
+      end
+      # JSON handles decimal numbers and hashes
+      begin
+        return JSON.parse(operand)
+      rescue JSON::ParserError
+        # Fall through to the number formats JSON does not accept
+      end
+      # Number formats JSON does not accept
+      begin
+        # Explicit base prefix: hex (0x), octal (0o) or binary (0b)
+        return Integer(operand) if operand.match?(/\A[+-]?0[xXoObB][0-9a-fA-F_]+\z/)
+        # Integers with underscore separators, e.g. 1_000. A leading zero is deliberately
+        # rejected because it is ambiguous: Ruby reads 010 as octal 8 while Python rejects it.
+        # Write 0o10 for octal 8 or 10 for decimal 10.
+        return Integer(operand, 10) if operand.match?(/\A[+-]?[1-9][0-9_]*\z/)
+        # Floats with underscore separators, e.g. 1_000.5. A leading zero is unambiguous here.
+        return Float(operand) if operand.match?(/\A[+-]?[0-9][0-9_]*(?:\.[0-9_]+(?:[eE][+-]?[0-9_]+)?|[eE][+-]?[0-9_]+)\z/)
+      rescue ArgumentError
+        # Fall through to the error cases
+      end
+
+      # A bare word is almost always a string the user forgot to quote
+      if operand.match?(/\A[A-Za-z_][A-Za-z0-9_]*\z/)
+        raise NameError, "Uninitialized constant #{operand}. Did you mean '#{operand}' as a string?"
+      end
+      raise "ERROR: Unable to parse operand: #{operand}"
+    end
+
+    # Processes the escape sequences in the contents of a double quoted operand
+    def unescape_double_quoted(string)
+      # A \xHH or octal escape can produce a byte which is not valid UTF-8 so build the result
+      # in binary. Otherwise preserve the encoding of the comparison the user passed in.
+      # The escape must not itself be escaped, e.g. "C:\\0" is a backslash followed by a zero.
+      binary = string.match?(/(?<!\\)(?:\\\\)*\\(?:x\h|[0-7])/)
+      result = binary ? ''.b : String.new(encoding: string.encoding)
+      string.scan(DOUBLE_QUOTE_TOKEN_REGEX).each do |token|
+        replacement =
+          if token.match?(INTERPOLATION_REGEX)
+            raise "ERROR: String interpolation is not supported in an operand: #{token}. " +
+                  'Interpolate in the script itself or escape it as \#' + token[1] +
+                  ' to compare against the literal text'
+          elsif !token.start_with?('\\')
+            token
+          elsif token.start_with?('\u{')
+            # \u{1F600} or \u{48 49} which is one or more whitespace separated codepoints
+            token[3..-2].split.map { |codepoint| pack_unicode_codepoint(codepoint, token) }.join
+          elsif token.length == 6 and token.start_with?('\u')
+            pack_unicode_codepoint(token[2..-1], token)
+          elsif token.length > 2 and token.start_with?('\x')
+            [token[2..-1].hex].pack('C')
+          elsif token.match?(/\A\\[0-7]/)
+            [token[1..-1].to_i(8) & 0xFF].pack('C')
+          elsif token.match?(UNSUPPORTED_ESCAPE_REGEX)
+            raise "ERROR: Unsupported escape sequence in operand: #{token}"
+          elsif token == '\u' or token == '\x'
+            raise "ERROR: Invalid escape sequence in operand: #{token}"
+          else
+            # Ruby drops the backslash of an escape which has no special meaning, e.g. "\q"
+            DOUBLE_QUOTE_ESCAPES.fetch(token[1], token[1])
+          end
+        result << (binary ? replacement.b : replacement)
+      end
+      return result
+    end
+
+    # Packs a single codepoint of a \u escape into a UTF-8 character. Ruby rejects a codepoint
+    # which is too long, above the largest Unicode character or in the UTF-16 surrogate range,
+    # so those raise here rather than silently packing into an invalid UTF-8 string.
+    def pack_unicode_codepoint(codepoint, token)
+      value = codepoint.hex
+      if codepoint.length > MAX_UNICODE_DIGITS or value > MAX_UNICODE_CODEPOINT or
+         UNICODE_SURROGATE_RANGE.include?(value)
+        raise "ERROR: Invalid Unicode codepoint in operand: #{token}"
+      end
+      return [value].pack('U')
+    end
+
+    # Splits the contents of a bracketed operand list on the top level commas.
+    # Quotes and nesting are tracked so "['a,b', [1, 2]]" is two elements, not four.
+    def split_operand_list(text)
+      elements = []
+      current = String.new(encoding: text.encoding)
+      depth = 0
+      quote = nil
+      escaped = false
+      text.each_char do |char|
+        if escaped
+          escaped = false
+        elsif quote
+          if char == '\\'
+            escaped = true
+          elsif char == quote
+            quote = nil
+          end
+        elsif char == "'" or char == '"'
+          quote = char
+        elsif char == '[' or char == '{'
+          depth += 1
+        elsif char == ']' or char == '}'
+          depth -= 1
+          raise "ERROR: Unable to parse operand: [#{text}]" if depth < 0
+        elsif char == ',' and depth == 0
+          elements << current.strip
+          current = String.new(encoding: text.encoding)
+          next
         end
+        current << char
       end
-      return [operator, operand]
+      raise "ERROR: Unable to parse operand: [#{text}]" if depth != 0 or quote
+
+      elements << current.strip
+      # A single empty element is an empty list, e.g. []
+      return [] if elements.length == 1 and elements[0].empty?
+      # A trailing comma is allowed, e.g. [1,], matching both Ruby and Python list literals
+      elements.pop if elements[-1].empty?
+      raise "ERROR: Unable to parse operand: [#{text}]" if elements.empty? or elements.any?(&:empty?)
+
+      return elements
+    end
+
+    # Compares a telemetry value against an operand using the given operator.
+    # Returns false rather than raising if the two values can not be compared.
+    def compare_values(value, operator, operand)
+      case operator
+      when "=="
+        value == operand
+      when "!="
+        value != operand
+      when ">"
+        value > operand
+      when ">="
+        value >= operand
+      when "<"
+        value < operand
+      when "<="
+        value <= operand
+      when "in"
+        # 'in' is containment against a list of values, matching Python
+        operand.is_a?(Array) && operand.include?(value)
+      else
+        raise "ERROR: Invalid operator: '#{operator}'"
+      end
+    rescue ArgumentError, NoMethodError, TypeError
+      # Comparing incompatible types, e.g. nil > 1, is simply not a match
+      false
     end
   end
 end