openc3 5.10.1 → 5.11.0

data/lib/openc3/script/web_socket_api.rb

@@ -190,7 +190,7 @@ module OpenC3
 
   # Base class for script-runner-api websockets - Do not use directly
   class ScriptWebSocketApi < WebSocketApi
-    def initialize(url: nil, write_timeout: 10.0, read_timeout: 10.0, connect_timeout: 5.0, authentication: nil, scope: $openc3_token)
+    def initialize(url: nil, write_timeout: 10.0, read_timeout: 10.0, connect_timeout: 5.0, authentication: nil, scope: $openc3_scope)
       url = generate_url() unless url
       super(url: url, write_timeout: write_timeout, read_timeout: read_timeout, connect_timeout: connect_timeout, authentication: authentication, scope: scope)
     end
@@ -217,23 +217,15 @@ module OpenC3
 
   # Log Messages WebSocket
   class MessagesWebSocketApi < CmdTlmWebSocketApi
-    def initialize(history_count: 0, url: nil, write_timeout: 10.0, read_timeout: 10.0, connect_timeout: 5.0, authentication: nil, scope: $openc3_scope)
+    def initialize(history_count: 0, start_time: nil, end_time: nil, severity: nil, types: nil, url: nil, write_timeout: 10.0, read_timeout: 10.0, connect_timeout: 5.0, authentication: nil, scope: $openc3_scope)
       @identifier = {
         channel: "MessagesChannel",
         history_count: history_count
       }
-      super(url: url, write_timeout: write_timeout, read_timeout: read_timeout, connect_timeout: connect_timeout, authentication: authentication, scope: scope)
-    end
-  end
-
-  # Notifications WebSocket
-  class NotificationsWebSocketApi < CmdTlmWebSocketApi
-    def initialize(history_count: 0, start_offset: nil, url: nil, write_timeout: 10.0, read_timeout: 10.0, connect_timeout: 5.0, authentication: nil, scope: $openc3_scope)
-      @identifier = {
-        channel: "NotificationsChannel",
-        history_count: history_count,
-        start_offset: start_offset
-      }
+      @identifier['start_time'] = start_time if start_time
+      @identifier['end_time'] = end_time if end_time
+      @identifier['severity'] = severity if severity
+      @identifier['types'] = types if types
       super(url: url, write_timeout: write_timeout, read_timeout: read_timeout, connect_timeout: connect_timeout, authentication: authentication, scope: scope)
     end
   end
@@ -425,3 +417,14 @@ end
 #
 # # Warning this saves all data to RAM. Do not use for large queries
 # data = OpenC3::StreamingWebSocketApi.read_all(items: ['DECOM__TLM__INST__HEALTH_STATUS__TEMP1__CONVERTED', 'DECOM__TLM__INST__HEALTH_STATUS__TEMP2__CONVERTED'], start_time: Time.now - 30, end_time: Time.now + 30)
+
+# $openc3_scope = 'DEFAULT'
+# OpenC3::MessagesWebSocketApi.new(history_count: 0, start_time: (Time.now - 86400).to_nsec_from_epoch, end_time: (Time.now - 60).to_nsec_from_epoch) do |api|
+#   500.times do
+#     # Note returns batch array
+#     data = api.read
+#     return if not data or data.length == 0
+#     puts "\nReceived #{data.length} log messages:"
+#     puts data
+#   end
+# end

data/lib/openc3/topics/telemetry_topic.rb

@@ -32,8 +32,9 @@ module OpenC3
         :target_name => packet.target_name,
         :packet_name => packet.packet_name,
         :received_count => packet.received_count,
-        :buffer => packet.buffer(false),
+        :buffer => packet.buffer(false)
       }
+      msg_hash[:extra] = JSON.generate(packet.extra.as_json, allow_nan: true) if packet.extra
       Topic.write_topic("#{scope}__TELEMETRY__{#{packet.target_name}}__#{packet.packet_name}", msg_hash)
     end
   end

data/lib/openc3/utilities/bucket_utilities.rb

@@ -58,6 +58,8 @@ module OpenC3
     # @param start_time [Time|nil] Ruby time to find files after. nil means no start (first file on).
     # @param end_time [Time|nil] Ruby time to find files before. nil means no end (up to last file).
     # @param overlap [Boolean] Whether to include files which overlap the start and end time
+    #   if true, file can be partially in the time range
+    #   if false, file must be completely in the time range
     # @param max_request [Integer] How many files to request in each API call
     # @param max_total [Integer] Total number of files before stopping API requests
     def self.files_between_time(bucket, prefix, start_time, end_time, file_suffix: nil,

data/lib/openc3/utilities/cli_generator.rb

@@ -51,7 +51,7 @@ module OpenC3
           FileUtils.mkdir(base_name) unless File.exist?(base_name)
           next
         end
-        output = ERB.new(File.read(file), trim_mode: "-").result(the_binding)
+        output = ERB.new(File.read(file).comment_erb(), trim_mode: "-").result(the_binding)
         File.open(base_name, 'w') do |file|
           file.write output
         end

data/lib/openc3/utilities/logger.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2023, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -44,11 +44,9 @@ module OpenC3
     # @return [String] Microservice name
     attr_reader :microservice_name
 
-    # @return [String] Scope
-    instance_attr_accessor :scope
-
     @@mutex = Mutex.new
     @@instance = nil
+    @@scope = ENV['OPENC3_SCOPE']
 
     # DEBUG only prints DEBUG messages
     DEBUG = ::Logger::DEBUG
@@ -67,11 +65,15 @@ module OpenC3
     ERROR_SEVERITY_STRING = 'ERROR'
     FATAL_SEVERITY_STRING = 'FATAL'
 
+    # Types
+    LOG = 'log'
+    NOTIFICATION = 'notification'
+    ALERT = 'alert'
+
     # @param level [Integer] The initial logging level
     def initialize(level = Logger::INFO)
       @stdout = true
       @level = level
-      @scope = nil
       @detail_string = nil
       @container_name = Socket.gethostname
       @microservice_name = nil
@@ -95,68 +97,53 @@ module OpenC3
     #   below the method name log level.
     # @param block [Proc] Block to call which should return a string to append
     #   to the log message
-    def debug(message = nil, scope: @scope, user: nil, &block)
-      log_message(DEBUG_SEVERITY_STRING, message, scope: scope, user: user, &block) if @level <= DEBUG
+    def debug(message = nil, scope: @@scope, user: nil, type: LOG, url: nil, &block)
+      log_message(DEBUG_SEVERITY_STRING, message, scope: scope, user: user, type: type, url: url, &block) if @level <= DEBUG
     end
 
     # (see #debug)
-    def info(message = nil, scope: @scope, user: nil, &block)
-      log_message(INFO_SEVERITY_STRING, message, scope: scope, user: user, &block) if @level <= INFO
+    def info(message = nil, scope: @@scope, user: nil, type: LOG, url: nil, &block)
+      log_message(INFO_SEVERITY_STRING, message, scope: scope, user: user, type: type, url: url, &block) if @level <= INFO
     end
 
     # (see #debug)
-    def warn(message = nil, scope: @scope, user: nil, &block)
-      log_message(WARN_SEVERITY_STRING, message, scope: scope, user: user, &block) if @level <= WARN
+    def warn(message = nil, scope: @@scope, user: nil, type: LOG, url: nil, &block)
+      log_message(WARN_SEVERITY_STRING, message, scope: scope, user: user, type: type, url: url, &block) if @level <= WARN
     end
 
     # (see #debug)
-    def error(message = nil, scope: @scope, user: nil, &block)
-      log_message(ERROR_SEVERITY_STRING, message, scope: scope, user: user, &block) if @level <= ERROR
+    def error(message = nil, scope: @@scope, user: nil, type: LOG, url: nil, &block)
+      log_message(ERROR_SEVERITY_STRING, message, scope: scope, user: user, type: type, url: url, &block) if @level <= ERROR
     end
 
     # (see #debug)
-    def fatal(message = nil, scope: @scope, user: nil, &block)
-      log_message(FATAL_SEVERITY_STRING, message, scope: scope, user: user, &block) if @level <= FATAL
+    def fatal(message = nil, scope: @@scope, user: nil, type: LOG, url: nil, &block)
+      log_message(FATAL_SEVERITY_STRING, message, scope: scope, user: user, type: type, url: url, &block) if @level <= FATAL
     end
 
     # (see #debug)
-    def self.debug(message = nil, scope: nil, user: nil, &block)
-      args = {}
-      args[:scope] = scope if scope
-      args[:user] = user if user
-      self.instance.debug(message, **args, &block)
+    def self.debug(message = nil, scope: @@scope, user: nil, type: LOG, url: nil, &block)
+      self.instance.debug(message, scope: scope, user: user, type: type, url: url, &block)
     end
 
     # (see #debug)
-    def self.info(message = nil, scope: nil, user: nil, &block)
-      args = {}
-      args[:scope] = scope if scope
-      args[:user] = user if user
-      self.instance.info(message, **args, &block)
+    def self.info(message = nil, scope: @@scope, user: nil, type: LOG, url: nil, &block)
+      self.instance.info(message, scope: scope, user: user, type: type, url: url, &block)
     end
 
     # (see #debug)
-    def self.warn(message = nil, scope: nil, user: nil, &block)
-      args = {}
-      args[:scope] = scope if scope
-      args[:user] = user if user
-      self.instance.warn(message, **args, &block)
+    def self.warn(message = nil, scope: @@scope, user: nil, type: LOG, url: nil, &block)
+      self.instance.warn(message, scope: scope, user: user, type: type, url: url, &block)
     end
 
     # (see #debug)
-    def self.error(message = nil, scope: nil, user: nil, &block)
-      args = {}
-      args[:scope] = scope if scope
-      args[:user] = user if user
-      self.instance.error(message, **args, &block)
+    def self.error(message = nil, scope: @@scope, user: nil, type: LOG, url: nil, &block)
+      self.instance.error(message, scope: scope, user: user, type: type, url: url, &block)
     end
 
     # (see #debug)
-    def self.fatal(message = nil, scope: nil, user: nil, &block)
-      args = {}
-      args[:scope] = scope if scope
-      args[:user] = user if user
-      self.instance.fatal(message, **args, &block)
+    def self.fatal(message = nil, scope: @@scope, user: nil, type: LOG, url: nil, &block)
+      self.instance.fatal(message, scope: scope, user: user, type: type, url: url, &block)
     end
 
     # @return [Logger] The logger instance
@@ -169,11 +156,28 @@ module OpenC3
       @@instance
     end
 
+    def self.scope
+      return @@scope
+    end
+
+    def self.scope=(scope)
+      @@scope = scope
+    end
+
+    def scope
+      return @@scope
+    end
+
+    def scope=(scope)
+      @@scope = scope
+    end
+
     protected
 
-    def log_message(severity_string, message, scope:, user:)
+    def log_message(severity_string, message, scope:, user:, type:, url:)
       @@mutex.synchronize do
-        data = { time: Time.now.to_nsec_from_epoch, '@timestamp' => Time.now.xmlschema(3), severity: severity_string }
+        time = Time.now
+        data = { time: time.to_nsec_from_epoch, '@timestamp' => time.xmlschema(3), severity: severity_string }
         data[:microservice_name] = @microservice_name if @microservice_name
         data[:detail] = @detail_string if @detail_string
         data[:user] = user['username'] || 'Unknown' if user # EE: If a user is passed, put its name ('Unknown' if it doesn't have a name). Don't include user data if no user was passed
@@ -182,17 +186,28 @@ module OpenC3
         end
         data[:container_name] = @container_name
         data[:log] = message
+        data[:type] = type
+        data[:url] = url if url
         if @stdout
-          puts data.as_json(:allow_nan => true).to_json(:allow_nan => true)
-          $stdout.flush
+          case severity_string
+          when WARN_SEVERITY_STRING, ERROR_SEVERITY_STRING, FATAL_SEVERITY_STRING
+            if ENV['OPENC3_LOG_STDERR']
+              $stderr.puts data.as_json(:allow_nan => true).to_json(:allow_nan => true)
+              $stderr.flush
+            else
+              $stdout.puts data.as_json(:allow_nan => true).to_json(:allow_nan => true)
+              $stdout.flush
+            end
+          else
+            $stdout.puts data.as_json(:allow_nan => true).to_json(:allow_nan => true)
+            $stdout.flush
+          end
         end
         unless @no_store
           if scope
             Topic.write_topic("#{scope}__openc3_log_messages", data)
           else
-            # The base openc3_log_messages doesn't have an associated logger
-            # so it must be limited to prevent unbounded stream growth
-            Topic.write_topic("openc3_log_messages", data, '*', 1000)
+            Topic.write_topic("NOSCOPE__openc3_log_messages", data)
           end
         end
       end

data/lib/openc3/utilities/metric.rb

@@ -1,6 +1,6 @@
 # encoding: ascii-8bit
 
-# Copyright 2022 OpenC3, Inc.
+# Copyright 2023 OpenC3, Inc.
 # All Rights Reserved.
 #
 # This program is free software; you can modify and/or redistribute it
@@ -38,6 +38,9 @@ module OpenC3
     # Sleeper used to delay update thread
     @@update_sleeper = nil
 
+    # Objects with a generate method to be called on each metric cycle (to generate metrics)
+    @@update_generators = []
+
     attr_reader :microservice
     attr_reader :scope
     attr_reader :data
@@ -83,7 +86,12 @@ module OpenC3
       @@update_sleeper = Sleeper.new
       while true
         start_time = Time.now
+
         @@mutex.synchronize do
+          @@update_generators.each do |generator|
+            generator.generate(@@instances[0])
+          end
+
           @@instances.each do |instance|
             instance.mutex.synchronize do
               json = {}
@@ -116,5 +124,15 @@ module OpenC3
 
     def graceful_kill
     end
+
+    def self.add_update_generator(object)
+      @@update_generators << object
+    end
   end
 end
+
+begin
+  require 'openc3-enterprise/utilities/metric'
+rescue LoadError
+  # Open Source Edition - Do nothing here
+end

data/lib/openc3/utilities/sleeper.rb

@@ -17,7 +17,7 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 module OpenC3
@@ -37,6 +37,7 @@ module OpenC3
     def sleep(seconds)
       read_ready, _ = IO.select(@readers, nil, nil, seconds)
       if read_ready && read_ready.include?(@pipe_reader)
+        @pipe_reader.close unless @pipe_reader.closed?
         return true
       else
         return false
@@ -48,6 +49,7 @@ module OpenC3
       if !@canceled
         @canceled = true
         @pipe_writer.write('.')
+        @pipe_writer.close
       end
     end
   end

data/lib/openc3/utilities/throttle.rb

@@ -0,0 +1,76 @@
+# encoding: ascii-8bit
+
+# Copyright 2023 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+# Usage:
+# throttle = OpenC3::Throttle.new(50.0) # Use max 50% cpu for work
+#
+# throttle.reset # Necessary if there are large periods of idle between hard work
+# 1000.times do
+#   throttle.start
+#   # Do one iteration of cpu itensive work here
+#   # complete will sleep if necessary to not use too much CPU
+#   throttle.complete
+# end
+
+module OpenC3
+  class Throttle
+    MIN_SLEEP_SECONDS = 0.001
+    MAX_SLEEP_SECONDS = 0.100
+
+    attr_reader :reset_time
+    attr_reader :total_sleep_time
+
+    # @param max_cpu_utilization [Float] 0.0-100.0
+    def initialize(max_cpu_utilization)
+      @max_cpu_utilization = Float(max_cpu_utilization)
+      raise ArgumentError "max_cpu_utilization must be between 0.0 and 100.0" if @max_cpu_utilization > 100.0 or @max_cpu_utilization < 0.0
+      @max_cpu_utilization /= 100.0 # Normalize
+      reset()
+    end
+
+    def reset
+      @reset_time = Time.now
+      @total_sleep_time = 0
+    end
+
+    def throttle_sleep
+      return if @max_cpu_utilization >= 1.0
+      total_time = Time.now - @reset_time
+      if total_time > 0
+        cpu_utilization = 1.0 - (@total_sleep_time / total_time)
+        if cpu_utilization > @max_cpu_utilization
+          # Need to throttle
+          # max_cpu_utilization = sleep_time + total_sleep_time
+          #                       ----------------------------
+          #                       total_time
+          #
+          # (max_cpu_utilization * total_time) = sleep_time + total_sleep_time
+          #
+          # sleep_time = (max_cpu_utilization * total_time) - total_sleep_time
+          #
+          sleep_time = (@max_cpu_utilization * total_time) - @total_sleep_time
+          if sleep_time > MIN_SLEEP_SECONDS
+            sleep_time = MAX_SLEEP_SECONDS if sleep_time > MAX_SLEEP_SECONDS
+            sleep(sleep_time)
+            @total_sleep_time += sleep_time
+          end
+        end
+      end
+    end
+  end
+end

data/lib/openc3/version.rb

@@ -1,14 +1,14 @@
 # encoding: ascii-8bit
 
-OPENC3_VERSION = '5.10.1'
+OPENC3_VERSION = '5.11.0'
 module OpenC3
   module Version
     MAJOR = '5'
-    MINOR = '10'
-    PATCH = '1'
+    MINOR = '11'
+    PATCH = '0'
     OTHER = ''
-    BUILD = 'b248056bc6fa7af79757e21f3b79601118eec0e5'
+    BUILD = 'f994eee0c2fadfc67ffbf0865ba798ed11ec1f5c'
   end
-  VERSION = '5.10.1'
-  GEM_VERSION = '5.10.1'
+  VERSION = '5.11.0'
+  GEM_VERSION = '5.11.0'
 end

data/templates/tool_angular/package.json

@@ -1,6 +1,6 @@
 {
   "name": "<%= tool_name %>",
-  "version": "5.10.1",
+  "version": "5.11.0",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",
@@ -12,36 +12,36 @@
   },
   "private": true,
   "dependencies": {
-    "@openc3/tool-common": "5.10.1",
-    "@angular/animations": "^16.2.0",
-    "@angular/cdk": "^16.2.0",
-    "@angular/common": "^16.2.0",
-    "@angular/compiler": "^16.2.0",
-    "@angular/core": "^16.2.0",
-    "@angular/forms": "^16.2.0",
-    "@angular/material": "16.2.0",
-    "@angular/platform-browser": "^16.2.0",
-    "@angular/platform-browser-dynamic": "^16.2.0",
-    "@angular/router": "^16.2.0",
+    "@openc3/tool-common": "5.11.0",
+    "@angular/animations": "^16.2.4",
+    "@angular/cdk": "^16.2.3",
+    "@angular/common": "^16.2.4",
+    "@angular/compiler": "^16.2.4",
+    "@angular/core": "^16.2.4",
+    "@angular/forms": "^16.2.4",
+    "@angular/material": "16.2.3",
+    "@angular/platform-browser": "^16.2.4",
+    "@angular/platform-browser-dynamic": "^16.2.4",
+    "@angular/router": "^16.2.4",
     "rxjs": "~7.8.0",
     "single-spa": ">=5.9.5",
     "single-spa-angular": "^8.1.0",
-    "tslib": "^2.6.1",
-    "zone.js": "~0.13.1"
+    "tslib": "^2.6.2",
+    "zone.js": "~0.13.3"
   },
   "devDependencies": {
-    "@angular-builders/custom-webpack": "16.0.0",
-    "@angular-devkit/build-angular": "^16.2.0",
-    "@angular/cli": "~16.2.0",
-    "@angular/compiler-cli": "^16.2.0",
+    "@angular-builders/custom-webpack": "16.0.1",
+    "@angular-devkit/build-angular": "^16.2.1",
+    "@angular/cli": "~16.2.1",
+    "@angular/compiler-cli": "^16.2.4",
     "@types/jasmine": "~4.3.0",
-    "jasmine-core": "~5.1.0",
+    "jasmine-core": "~5.1.1",
     "karma": "~6.4.0",
     "karma-chrome-launcher": "~3.2.0",
     "karma-coverage": "~2.2.0",
     "karma-jasmine": "~5.1.0",
     "karma-jasmine-html-reporter": "~2.1.0",
     "style-loader": "^3.3.1",
-    "typescript": "~5.1.6"
+    "typescript": "~5.2.2"
   }
 }