openc3 5.10.0 → 5.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c37c79f2062a5921e13c3370ce281c443b9abba929ca36f152df7cd1bcea92e
-  data.tar.gz: 643d74e05f61eb9016eaefbe5dca61316939e3d4e063fcf4150423cee4d95474
+  metadata.gz: beada8d3ae31acc7a45fff23d454f4667b552a5de293f6c17f6da72626dd8490
+  data.tar.gz: 950b07bf2f6a3173284d41a77ae7abf4cf6f9582a885dbedc585d50001fadcd8
 SHA512:
-  metadata.gz: 25285376d600c16a3d0b6d485a359998fda7226fc3781f639cdde992370291fa7d23b0ce6ca89e02ddcf4e6386dd596839ed4d8c704003b893e37e726ed1a7e4
-  data.tar.gz: 3ae81b7ea67047a8e23cfb161aeb77b4e7224e44b2c9c1dc4fa2dd78cb06eae50710b0a5c882cfcaf8da9e0214b6b96e517ec2a6af0b7671f5b8e8a0dc1448fe
+  metadata.gz: 7b0e9f874d5a88c167ba9bd78c3718d1b646ab80560106e01fb109189c27d6b399a3f51531b23815129ea24f4b9194770c9448d8e4be832c0926a6f5b8c3f9ad
+  data.tar.gz: e5c20e0648e27b311de33c0d06eb27bb229110eac0fcf197619dac6a3232b239334cbda07bf3f79e88ddb7425b773191c04f9929883a6042b36633c99924285a

data/Gemfile

@@ -14,4 +14,4 @@ gem 'tzinfo-data'
 gem 'rspec-rails', '~> 6.0'
 gem 'simplecov', '~> 0.20'
 gem 'simplecov-cobertura', '~> 2.1'
-gem 'mock_redis'
+gem 'mock_redis', '0.36'

data/Rakefile

@@ -17,7 +17,7 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 require 'open3'
@@ -63,7 +63,8 @@ task :build => [:devkit] do
       'telemetry',
       'packet',
       'platform',
-      'buffered_file'
+      'buffered_file',
+      'reducer_microservice'
     ]
 
     extensions.each do |extension_name|

data/data/config/target.yaml

@@ -156,6 +156,15 @@ TARGET:
           required: true
           description: Number of seconds between runs of the cleanup process (default = 900 = 15 minutes)
           values: \d+
+    REDUCER_DISABLE:
+      summary: Disables the data reduction microservice for the target
+    REDUCER_MAX_CPU_UTILIZATION:
+      summary: Maximum amount of CPU utilization to apply to data reduction
+      parameters:
+        - name: Percentage
+          required: true
+          description: 0 to 100 percent (default = 30)
+          values: \d+
     TARGET_MICROSERVICE:
       summary: Breaks a target microservice out into its own process.
       description:

data/ext/openc3/ext/packet/packet.c

@@ -60,6 +60,7 @@ static ID id_ivar_description = 0;
 static ID id_ivar_stored = 0;
 static ID id_ivar_extra = 0;
 static ID id_ivar_template = 0;
+static ID id_ivar_packet_time = 0;
 
 /* Sets the target name this packet is associated with. Unidentified packets
  * will have target name set to nil.
@@ -283,6 +284,7 @@ static VALUE packet_initialize(int argc, VALUE *argv, VALUE self)
   rb_ivar_set(self, id_ivar_stored, Qfalse);
   rb_ivar_set(self, id_ivar_extra, Qnil);
   rb_ivar_set(self, id_ivar_template, Qnil);
+  rb_ivar_set(self, id_ivar_packet_time, Qnil);
   return self;
 }
 
@@ -323,6 +325,7 @@ void Init_packet(void)
   id_ivar_stored = rb_intern("@stored");
   id_ivar_extra = rb_intern("@extra");
   id_ivar_template = rb_intern("@template");
+  id_ivar_packet_time = rb_intern("@packet_time");
 
   cPacket = rb_define_class_under(mOpenC3, "Packet", cStructure);
   rb_define_method(cPacket, "initialize", packet_initialize, -1);

data/ext/openc3/ext/reducer_microservice/extconf.rb

@@ -0,0 +1,13 @@
+require 'mkmf'
+
+unless $CFLAGS.gsub!(/ -O[\dsz]?/, ' -O3')
+  $CFLAGS << ' -O3'
+end
+if /gcc/.match?(CONFIG['CC'])
+  $CFLAGS << ' -Wall'
+  if $DEBUG && !$CFLAGS.gsub!(/ -O[\dsz]?/, ' -O0 -ggdb')
+    $CFLAGS << ' -O0 -ggdb'
+  end
+end
+
+create_makefile 'openc3/ext/reducer_microservice'

data/ext/openc3/ext/reducer_microservice/reducer_microservice.c

@@ -0,0 +1,165 @@
+/*
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+*/
+
+/*
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+*/
+
+#include "ruby.h"
+#include "stdio.h"
+#include "math.h"
+
+VALUE mOpenC3;
+VALUE cMicroservice;
+VALUE cReducerMicroservice;
+
+static ID id_method_raw_values = 0;
+static ID id_method_converted_values = 0;
+static ID id_method_less_than = 0;
+static ID id_method_greater_than = 0;
+
+static int reducer_microservice_update_min_stats_raw_value(VALUE key, VALUE value, VALUE reduced)
+{
+  volatile VALUE vals_key = Qnil;
+  volatile VALUE n_key = Qnil;
+  volatile VALUE x_key = Qnil;
+  volatile VALUE reduced_entry = Qnil;
+
+  if (RTEST(value))
+  {
+    vals_key = rb_str_new(RSTRING_PTR(key), RSTRING_LEN(key));
+    rb_str_append(vals_key, rb_str_new2("__VALS"));
+    reduced_entry = rb_hash_aref(reduced, vals_key);
+    if (!(RTEST(reduced_entry)))
+    {
+      reduced_entry = rb_ary_new();
+      rb_hash_aset(reduced, vals_key, reduced_entry);
+    }
+    rb_ary_push(reduced_entry, value);
+
+    n_key = rb_str_new(RSTRING_PTR(key), RSTRING_LEN(key));
+    rb_str_append(n_key, rb_str_new2("__N"));
+    reduced_entry = rb_hash_aref(reduced, n_key);
+    if (!(RTEST(reduced_entry)))
+    {
+      reduced_entry = value;
+      rb_hash_aset(reduced, n_key, value);
+    }
+    if (RTEST(rb_funcall(value, id_method_less_than, 1, reduced_entry)))
+    {
+      rb_hash_aset(reduced, n_key, value);
+    }
+
+    x_key = rb_str_new(RSTRING_PTR(key), RSTRING_LEN(key));
+    rb_str_append(x_key, rb_str_new2("__X"));
+    reduced_entry = rb_hash_aref(reduced, x_key);
+    if (!(RTEST(reduced_entry)))
+    {
+      reduced_entry = value;
+      rb_hash_aset(reduced, x_key, value);
+    }
+    if (RTEST(rb_funcall(value, id_method_greater_than, 1, reduced_entry)))
+    {
+      rb_hash_aset(reduced, x_key, value);
+    }
+  }
+
+  return ST_CONTINUE;
+}
+
+static int reducer_microservice_update_min_stats_converted_value(VALUE key, VALUE value, VALUE reduced)
+{
+  volatile VALUE cvals_key = Qnil;
+  volatile VALUE cn_key = Qnil;
+  volatile VALUE cx_key = Qnil;
+  volatile VALUE reduced_entry = Qnil;
+
+  if (RTEST(value))
+  {
+    cvals_key = rb_str_new(RSTRING_PTR(key), RSTRING_LEN(key));
+    rb_str_append(cvals_key, rb_str_new2("__CVALS"));
+    reduced_entry = rb_hash_aref(reduced, cvals_key);
+    if (!(RTEST(reduced_entry)))
+    {
+      reduced_entry = rb_ary_new();
+      rb_hash_aset(reduced, cvals_key, reduced_entry);
+    }
+    rb_ary_push(reduced_entry, value);
+
+    cn_key = rb_str_new(RSTRING_PTR(key), RSTRING_LEN(key));
+    rb_str_append(cn_key, rb_str_new2("__CN"));
+    reduced_entry = rb_hash_aref(reduced, cn_key);
+    if (!(RTEST(reduced_entry)))
+    {
+      reduced_entry = value;
+      rb_hash_aset(reduced, cn_key, value);
+    }
+    if (RTEST(rb_funcall(value, id_method_less_than, 1, reduced_entry)))
+    {
+      rb_hash_aset(reduced, cn_key, value);
+    }
+
+    cx_key = rb_str_new(RSTRING_PTR(key), RSTRING_LEN(key));
+    rb_str_append(cx_key, rb_str_new2("__CX"));
+    reduced_entry = rb_hash_aref(reduced, cx_key);
+    if (!(RTEST(reduced_entry)))
+    {
+      reduced_entry = value;
+      rb_hash_aset(reduced, cx_key, value);
+    }
+    if (RTEST(rb_funcall(value, id_method_greater_than, 1, reduced_entry)))
+    {
+      rb_hash_aset(reduced, cx_key, value);
+    }
+  }
+
+  return ST_CONTINUE;
+}
+
+static VALUE reducer_microservice_update_min_stats(VALUE self, VALUE reduced, VALUE state)
+{
+  volatile VALUE raw_values = rb_funcall(state, id_method_raw_values, 0);
+  volatile VALUE converted_values = rb_funcall(state, id_method_converted_values, 0);
+
+  /* Update statistics for this packet's raw values */
+  rb_hash_foreach(raw_values, reducer_microservice_update_min_stats_raw_value, reduced);
+
+  /* Update statistics for this packet's converted values */
+  rb_hash_foreach(converted_values, reducer_microservice_update_min_stats_converted_value, reduced);
+
+  return Qnil;
+}
+
+/*
+ * Initialize methods for ReducerMicroservice
+ */
+void Init_reducer_microservice(void)
+{
+  id_method_raw_values = rb_intern("raw_values");
+  id_method_converted_values = rb_intern("converted_values");
+  id_method_less_than = rb_intern("<");
+  id_method_greater_than = rb_intern(">");
+
+  mOpenC3 = rb_define_module("OpenC3");
+  rb_require("openc3/conversions/conversion");
+  cMicroservice = rb_const_get(mOpenC3, rb_intern("Microservice"));
+  cReducerMicroservice = rb_define_class_under(mOpenC3, "ReducerMicroservice", cMicroservice);
+  rb_define_method(cReducerMicroservice, "update_min_stats", reducer_microservice_update_min_stats, 2);
+}

data/ext/openc3/ext/structure/structure.c

@@ -18,7 +18,7 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 */
 
@@ -64,6 +64,8 @@ static ID id_method_Integer = 0;
 static ID id_method_Float = 0;
 static ID id_method_kind_of = 0;
 static ID id_method_allocate_buffer_if_needed = 0;
+static ID id_method_new = 0;
+static ID id_method_class2 = 0;
 
 static ID id_ivar_buffer = 0;
 static ID id_ivar_bit_offset = 0;
@@ -1304,21 +1306,15 @@ static VALUE structure_length(VALUE self)
 
 static VALUE read_item_internal(VALUE self, VALUE item, VALUE buffer)
 {
-  volatile VALUE data_type = Qnil;
   volatile VALUE accessor = Qnil;
 
-  data_type = rb_ivar_get(item, id_ivar_data_type);
-  if (data_type == symbol_DERIVED)
-  {
-    return Qnil;
-  }
+  accessor = rb_ivar_get(self, id_ivar_accessor);
 
   if (!(RTEST(buffer)))
   {
     buffer = rb_funcall(self, id_method_allocate_buffer_if_needed, 0);
   }
 
-  accessor = rb_ivar_get(self, id_ivar_accessor);
   return rb_funcall(accessor, id_method_read_item, 2, item, buffer);
 }
 
@@ -1556,7 +1552,7 @@ static VALUE structure_initialize(int argc, VALUE *argv, VALUE self)
     rb_ivar_set(self, id_ivar_fixed_size, Qtrue);
     rb_ivar_set(self, id_ivar_short_buffer_allowed, Qfalse);
     rb_ivar_set(self, id_ivar_mutex, Qnil);
-    rb_ivar_set(self, id_ivar_accessor, cBinaryAccessor);
+    rb_ivar_set(self, id_ivar_accessor, rb_funcall(cBinaryAccessor, id_method_new, 1, self));
   }
   else
   {
@@ -1605,6 +1601,7 @@ void Init_structure(void)
   cAccessor = rb_define_class_under(mOpenC3, "Accessor", rb_cObject);
   cBinaryAccessor = rb_define_class_under(mOpenC3, "BinaryAccessor", cAccessor);
 
+  id_method_class2 = rb_intern("class");
   id_method_to_s = rb_intern("to_s");
   id_method_raise_buffer_error = rb_intern("raise_buffer_error");
   id_method_read_array = rb_intern("read_array");
@@ -1617,6 +1614,7 @@ void Init_structure(void)
   id_method_Float = rb_intern("Float");
   id_method_kind_of = rb_intern("kind_of?");
   id_method_allocate_buffer_if_needed = rb_intern("allocate_buffer_if_needed");
+  id_method_new = rb_intern("new");
 
   MIN_INT8 = INT2NUM(-128);
   MAX_INT8 = INT2NUM(127);

data/lib/openc3/accessors/accessor.rb

@@ -13,16 +13,66 @@
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU Affero General Public License for more details.
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 module OpenC3
   class Accessor
+    attr_accessor :packet
+
+    def initialize(packet = nil)
+      @packet = packet
+      @args = []
+    end
+
     def read_item(item, buffer)
-      raise "Must be defined by subclass"
+      self.class.read_item(item, buffer)
     end
 
     def write_item(item, value, buffer)
+      self.class.write_item(item, value, buffer)
+    end
+
+    def read_items(items, buffer)
+      result = {}
+      items.each do |item|
+        result[item.name] = read_item(item, buffer)
+      end
+      return result
+    end
+
+    def write_items(items, values, buffer)
+      items.each_with_index do |item, index|
+        write_item(item, values[index], buffer)
+      end
+      return values
+    end
+
+    def args
+      return @args
+    end
+
+    def enforce_encoding
+      return 'ASCII-8BIT'.freeze
+    end
+
+    def enforce_length
+      return true
+    end
+
+    def enforce_short_buffer_allowed
+      return false
+    end
+
+    def enforce_derived_write_conversion(item)
+      return true
+    end
+
+    def self.read_item(item, buffer)
+      raise "Must be defined by subclass"
+    end
+
+    def self.write_item(item, value, buffer)
       raise "Must be defined by subclass"
     end
 
@@ -38,7 +88,7 @@ module OpenC3
       items.each_with_index do |item, index|
         write_item(item, values[index], buffer)
       end
-      return buffer
+      return values
     end
 
     def self.convert_to_type(value, item)

data/lib/openc3/accessors/binary_accessor.rb

@@ -1228,5 +1228,21 @@ module OpenC3
       end
       values
     end
+
+    def enforce_encoding
+      return 'ASCII-8BIT'.freeze
+    end
+
+    def enforce_length
+      return true
+    end
+
+    def enforce_short_buffer_allowed
+      return false
+    end
+
+    def enforce_derived_write_conversion(item)
+      return true
+    end
   end # class BinaryAccessor
 end

data/lib/openc3/accessors/cbor_accessor.rb

@@ -13,7 +13,7 @@
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU Affero General Public License for more details.
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 require 'cbor'
@@ -49,7 +49,7 @@ module OpenC3
         buffer.replace(decoded.to_cbor)
       end
 
-      return buffer
+      return value
     end
 
     def self.read_items(items, buffer)
@@ -79,7 +79,7 @@ module OpenC3
         buffer.replace(decoded.to_cbor)
       end
 
-      return buffer
+      return values
     end
 
   end

data/lib/openc3/accessors/form_accessor.rb

@@ -0,0 +1,78 @@
+# encoding: ascii-8bit
+
+# Copyright 2023 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/accessors/accessor'
+require 'uri'
+
+module OpenC3
+  class FormAccessor < Accessor
+    def self.read_item(item, buffer)
+      ary = URI.decode_www_form(buffer)
+      value = nil
+      ary.each do |key, ary_value|
+        if key == item.key
+          if value
+            if not Array === value
+              value_temp = []
+              value_temp << value
+              value = value_temp
+            end
+            value << ary_value
+          else
+            value = ary_value
+          end
+        end
+      end
+      return value
+    end
+
+    def self.write_item(item, value, buffer)
+      ary = URI.decode_www_form(buffer)
+
+      # Remove existing item and bad keys from array
+      ary.reject! {|key, ary_value| (key == item.key) or (key.to_s[0] == "\u0000")}
+
+      if Array === value
+        value.each do |value_value|
+          ary << [item.key, value_value]
+        end
+      else
+        ary << [item.key, value]
+      end
+
+      buffer.replace(URI.encode_www_form(ary))
+      return value
+    end
+
+    def enforce_encoding
+      return nil
+    end
+
+    def enforce_length
+      return false
+    end
+
+    def enforce_short_buffer_allowed
+      return true
+    end
+
+    def enforce_derived_write_conversion(item)
+      return true
+    end
+  end
+end

data/lib/openc3/accessors/http_accessor.rb

@@ -0,0 +1,145 @@
+# encoding: ascii-8bit
+
+# Copyright 2023 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/accessors/accessor'
+
+module OpenC3
+  class HttpAccessor < Accessor
+    def initialize(packet, body_accessor = 'FormAccessor', *body_accessor_args)
+      super(packet)
+      @args << body_accessor
+      body_accessor_args.each do |arg|
+        @args << arg
+      end
+      klass = OpenC3.require_class(body_accessor)
+      @body_accessor = klass.new(packet, *body_accessor_args)
+    end
+
+    def read_item(item, buffer)
+      item_name = item.name
+      case item_name
+      when 'HTTP_STATUS'
+        return nil unless @packet.extra
+        return @packet.extra['HTTP_STATUS']
+      when 'HTTP_PATH'
+        return nil unless @packet.extra
+        return @packet.extra['HTTP_PATH']
+      when 'HTTP_METHOD'
+        return nil unless @packet.extra
+        return @packet.extra['HTTP_METHOD']
+      when 'HTTP_PACKET'
+        return nil unless @packet.extra
+        return @packet.extra['HTTP_PACKET']
+      when 'HTTP_ERROR_PACKET'
+        return nil unless @packet.extra
+        return @packet.extra['HTTP_ERROR_PACKET']
+      when /^HTTP_QUERY_/
+        return nil unless @packet.extra
+        if item.key
+          query_name = item.key
+        else
+          query_name = item_name[11..-1]
+        end
+        queries = @packet.extra['HTTP_QUERIES']
+        if queries
+          return queries[query_name]
+        else
+          return nil
+        end
+      when /^HTTP_HEADER_/
+        return nil unless @packet.extra
+        if item.key
+          header_name = item.key
+        else
+          header_name = item_name[12..-1]
+        end
+        headers = @packet.extra['HTTP_HEADERS']
+        if headers
+          return headers[header_name]
+        else
+          return nil
+        end
+      else
+        return @body_accessor.read_item(item, buffer)
+      end
+    end
+
+    def write_item(item, value, buffer)
+      item_name = item.name
+      case item_name
+      when 'HTTP_STATUS'
+        @packet.extra ||= {}
+        @packet.extra['HTTP_STATUS'] = value.to_i
+      when 'HTTP_PATH'
+        @packet.extra ||= {}
+        @packet.extra['HTTP_PATH'] = value.to_s
+      when 'HTTP_METHOD'
+        @packet.extra ||= {}
+        @packet.extra['HTTP_METHOD'] = value.to_s.downcase
+      when 'HTTP_PACKET'
+        @packet.extra ||= {}
+        @packet.extra['HTTP_PACKET'] = value.to_s.upcase
+      when 'HTTP_ERROR_PACKET'
+        @packet.extra ||= {}
+        @packet.extra['HTTP_ERROR_PACKET'] = value.to_s.upcase
+      when /^HTTP_QUERY_/
+        @packet.extra ||= {}
+        if item.key
+          query_name = item.key
+        else
+          query_name = item_name[11..-1]
+        end
+        queries = @packet.extra['HTTP_QUERIES'] ||= {}
+        queries[query_name] = value.to_s
+      when /^HTTP_HEADER_/
+        @packet.extra ||= {}
+        if item.key
+          header_name = item.key
+        else
+          header_name = item_name[12..-1]
+        end
+        headers = @packet.extra['HTTP_HEADERS'] ||= {}
+        headers[header_name] = value.to_s
+      else
+        @body_accessor.write_item(item, value, buffer)
+      end
+      return value
+    end
+
+    def enforce_encoding
+      return @body_accessor.enforce_encoding
+    end
+
+    def enforce_length
+      return @body_accessor.enforce_length
+    end
+
+    def enforce_short_buffer_allowed
+      return @body_accessor.enforce_short_buffer_allowed
+    end
+
+    def enforce_derived_write_conversion(item)
+      case item.name
+      when 'HTTP_STATUS', 'HTTP_PATH', 'HTTP_METHOD', 'HTTP_PACKET', 'HTTP_ERROR_PACKET', /^HTTP_QUERY_/, /^HTTP_HEADER_/
+        return false
+      else
+        return @body_accessor.enforce_derived_write_conversion
+      end
+    end
+  end
+end