openc3 5.1.1 → 5.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/bin/openc3cli +48 -9
  3. data/data/config/interface_modifiers.yaml +14 -0
  4. data/data/config/parameter_modifiers.yaml +5 -3
  5. data/data/config/screen.yaml +12 -8
  6. data/data/config/target.yaml +33 -0
  7. data/ext/openc3/ext/config_parser/config_parser.c +66 -63
  8. data/ext/openc3/ext/packet/packet.c +1 -4
  9. data/lib/openc3/api/README.md +5 -0
  10. data/lib/openc3/api/api.rb +3 -1
  11. data/lib/openc3/api/cmd_api.rb +43 -112
  12. data/lib/openc3/api/interface_api.rb +3 -3
  13. data/lib/openc3/api/offline_access_api.rb +78 -0
  14. data/lib/openc3/api/settings_api.rb +3 -1
  15. data/lib/openc3/api/stash_api.rb +63 -0
  16. data/lib/openc3/api/target_api.rb +4 -5
  17. data/lib/openc3/config/config_parser.rb +47 -47
  18. data/lib/openc3/interfaces/interface.rb +11 -1
  19. data/lib/openc3/interfaces/protocols/burst_protocol.rb +30 -16
  20. data/lib/openc3/interfaces/protocols/fixed_protocol.rb +8 -2
  21. data/lib/openc3/interfaces/protocols/ignore_packet_protocol.rb +2 -2
  22. data/lib/openc3/interfaces/protocols/override_protocol.rb +2 -2
  23. data/lib/openc3/interfaces/tcpip_server_interface.rb +3 -1
  24. data/lib/openc3/io/json_api_object.rb +30 -9
  25. data/lib/openc3/io/json_drb.rb +6 -1
  26. data/lib/openc3/io/json_drb_object.rb +18 -9
  27. data/lib/openc3/io/json_rpc.rb +5 -3
  28. data/lib/openc3/logs/buffered_packet_log_writer.rb +1 -1
  29. data/lib/openc3/logs/log_writer.rb +8 -2
  30. data/lib/openc3/microservices/cleanup_microservice.rb +3 -3
  31. data/lib/openc3/microservices/decom_microservice.rb +8 -8
  32. data/lib/openc3/microservices/interface_microservice.rb +86 -71
  33. data/lib/openc3/microservices/log_microservice.rb +5 -3
  34. data/lib/openc3/microservices/microservice.rb +18 -14
  35. data/lib/openc3/microservices/multi_microservice.rb +62 -0
  36. data/lib/openc3/microservices/periodic_microservice.rb +58 -0
  37. data/lib/openc3/microservices/reaction_microservice.rb +61 -47
  38. data/lib/openc3/microservices/reducer_microservice.rb +64 -40
  39. data/lib/openc3/microservices/router_microservice.rb +4 -4
  40. data/lib/openc3/microservices/text_log_microservice.rb +2 -2
  41. data/lib/openc3/microservices/timeline_microservice.rb +44 -30
  42. data/lib/openc3/microservices/trigger_group_microservice.rb +39 -36
  43. data/lib/openc3/migrations/20221202214600_add_target_names.rb +30 -0
  44. data/lib/openc3/migrations/20221210174900_convert_to_multi.rb +65 -0
  45. data/lib/openc3/models/cvt_model.rb +1 -1
  46. data/lib/openc3/models/gem_model.rb +24 -20
  47. data/lib/openc3/models/interface_model.rb +69 -35
  48. data/lib/openc3/models/metadata_model.rb +1 -1
  49. data/lib/openc3/models/microservice_model.rb +7 -24
  50. data/lib/openc3/models/migration_model.rb +52 -0
  51. data/lib/openc3/models/model.rb +2 -7
  52. data/lib/openc3/models/note_model.rb +1 -1
  53. data/lib/openc3/models/offline_access_model.rb +55 -0
  54. data/lib/openc3/models/plugin_model.rb +12 -3
  55. data/lib/openc3/models/reaction_model.rb +6 -2
  56. data/lib/openc3/models/scope_model.rb +89 -13
  57. data/lib/openc3/models/settings_model.rb +1 -1
  58. data/lib/openc3/models/stash_model.rb +53 -0
  59. data/lib/openc3/models/target_model.rb +301 -130
  60. data/lib/openc3/models/tool_model.rb +1 -12
  61. data/lib/openc3/models/widget_model.rb +1 -6
  62. data/lib/openc3/operators/microservice_operator.rb +45 -6
  63. data/lib/openc3/operators/operator.rb +27 -5
  64. data/lib/openc3/packets/commands.rb +1 -25
  65. data/lib/openc3/packets/limits.rb +0 -75
  66. data/lib/openc3/packets/packet.rb +0 -28
  67. data/lib/openc3/packets/packet_item.rb +23 -0
  68. data/lib/openc3/packets/packet_item_limits.rb +2 -2
  69. data/lib/openc3/packets/parsers/state_parser.rb +10 -6
  70. data/lib/openc3/packets/telemetry.rb +1 -45
  71. data/lib/openc3/script/commands.rb +41 -71
  72. data/lib/openc3/script/extract.rb +15 -1
  73. data/lib/openc3/script/{calendar.rb → metadata.rb} +42 -17
  74. data/lib/openc3/script/script.rb +13 -5
  75. data/lib/openc3/script/storage.rb +3 -1
  76. data/lib/openc3/system/system.rb +19 -17
  77. data/lib/openc3/tools/cmd_tlm_server/interface_thread.rb +4 -4
  78. data/lib/openc3/top_level.rb +3 -3
  79. data/lib/openc3/topics/command_decom_topic.rb +2 -2
  80. data/lib/openc3/topics/command_topic.rb +7 -6
  81. data/lib/openc3/topics/interface_topic.rb +2 -2
  82. data/lib/openc3/topics/router_topic.rb +1 -1
  83. data/lib/openc3/topics/telemetry_topic.rb +2 -1
  84. data/lib/openc3/utilities/authentication.rb +35 -14
  85. data/lib/openc3/utilities/aws_bucket.rb +4 -3
  86. data/lib/openc3/utilities/bucket.rb +4 -2
  87. data/lib/openc3/utilities/bucket_file_cache.rb +3 -8
  88. data/lib/openc3/utilities/bucket_utilities.rb +77 -15
  89. data/lib/openc3/utilities/local_mode.rb +12 -9
  90. data/lib/openc3/utilities/logger.rb +17 -9
  91. data/lib/openc3/utilities/message_log.rb +6 -5
  92. data/lib/openc3/utilities/migration.rb +22 -0
  93. data/lib/openc3/utilities/store_autoload.rb +7 -5
  94. data/lib/openc3/utilities/target_file.rb +9 -7
  95. data/lib/openc3/version.rb +6 -6
  96. data/lib/openc3.rb +2 -1
  97. metadata +14 -3
@@ -17,7 +17,7 @@
17
17
  # All changes Copyright 2022, OpenC3, Inc.
18
18
  # All Rights Reserved
19
19
  #
20
- # This file may also be used under the terms of a commercial license
20
+ # This file may also be used under the terms of a commercial license
21
21
  # if purchased from OpenC3, Inc.
22
22
 
23
23
  require 'openc3/version'
@@ -60,6 +60,8 @@ module OpenC3
60
60
 
61
61
  REFRESH_OFFSET_SECONDS = 60
62
62
 
63
+ attr_reader :refresh_token
64
+
63
65
  # @param url [String] The url of the openc3 or keycloak in the cluster
64
66
  def initialize(url)
65
67
  @url = url
@@ -72,7 +74,7 @@ module OpenC3
72
74
  end
73
75
 
74
76
  # Load the token from the environment
75
- def token()
77
+ def token
76
78
  @auth_mutex.synchronize do
77
79
  @log = [nil, nil]
78
80
  current_time = Time.now.to_i
@@ -87,23 +89,42 @@ module OpenC3
87
89
  "Bearer #{@token}"
88
90
  end
89
91
 
92
+ def get_token_from_refresh_token(refresh_token)
93
+ current_time = Time.now.to_i
94
+ begin
95
+ @refresh_token = refresh_token
96
+ _refresh_token(current_time)
97
+ return @token
98
+ rescue OpenC3AuthenticationError
99
+ return nil
100
+ end
101
+ return nil
102
+ end
103
+
90
104
  private
91
105
 
92
106
  # Make the token and save token to instance
93
107
  def _make_token(current_time)
94
108
  client_id = ENV['OPENC3_API_CLIENT'] || 'api'
95
- data = "username=#{ENV['OPENC3_API_USER']}&password=#{ENV['OPENC3_API_PASSWORD']}"
96
- data << "&client_id=#{client_id}"
97
- data << '&grant_type=password&scope=openid'
98
- headers = {
99
- 'Content-Type' => 'application/x-www-form-urlencoded',
100
- 'User-Agent' => "OpenC3KeycloakAuthorization / #{OPENC3_VERSION} (ruby/openc3/lib/utilities/authentication)",
101
- }
102
- oath = _make_request(headers, data)
103
- @token = oath['access_token']
104
- @refresh_token = oath['refresh_token']
105
- @expires_at = current_time + oath['expires_in'] - REFRESH_OFFSET_SECONDS
106
- @refresh_expires_at = current_time + oath['refresh_expires_in'] - REFRESH_OFFSET_SECONDS
109
+ if ENV['OPENC3_API_USER'] and ENV['OPENC3_API_PASSWORD']
110
+ # Username and password
111
+ data = "username=#{ENV['OPENC3_API_USER']}&password=#{ENV['OPENC3_API_PASSWORD']}"
112
+ data << "&client_id=#{client_id}"
113
+ data << '&grant_type=password&scope=openid'
114
+ headers = {
115
+ 'Content-Type' => 'application/x-www-form-urlencoded',
116
+ 'User-Agent' => "OpenC3KeycloakAuthorization / #{OPENC3_VERSION} (ruby/openc3/lib/utilities/authentication)",
117
+ }
118
+ oath = _make_request(headers, data)
119
+ @token = oath['access_token']
120
+ @refresh_token = oath['refresh_token']
121
+ @expires_at = current_time + oath['expires_in'] - REFRESH_OFFSET_SECONDS
122
+ @refresh_expires_at = current_time + oath['refresh_expires_in'] - REFRESH_OFFSET_SECONDS
123
+ else
124
+ # Offline Access Token
125
+ @refresh_token ||= ENV['OPENC3_API_TOKEN']
126
+ _refresh_token(current_time)
127
+ end
107
128
  end
108
129
 
109
130
  # Refresh the token and save token to instance
@@ -13,7 +13,7 @@
13
13
  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
14
  # GNU Affero General Public License for more details.
15
15
  #
16
- # This file may also be used under the terms of a commercial license
16
+ # This file may also be used under the terms of a commercial license
17
17
  # if purchased from OpenC3, Inc.
18
18
 
19
19
  require 'openc3/utilities/bucket'
@@ -107,12 +107,13 @@ module OpenC3
107
107
  nil
108
108
  end
109
109
 
110
- def list_objects(bucket:, prefix: nil)
110
+ def list_objects(bucket:, prefix: nil, max_request: 1000, max_total: 100_000)
111
111
  token = nil
112
112
  result = []
113
113
  while true
114
- resp = @client.list_objects_v2(bucket: bucket, prefix: prefix, max_keys: 1000)
114
+ resp = @client.list_objects_v2(bucket: bucket, prefix: prefix, max_keys: max_request)
115
115
  result.concat(resp.contents)
116
+ break if result.length >= max_total
116
117
  break unless resp.is_truncated
117
118
  token = resp.next_continuation_token
118
119
  end
@@ -13,9 +13,11 @@
13
13
  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
14
  # GNU Affero General Public License for more details.
15
15
  #
16
- # This file may also be used under the terms of a commercial license
16
+ # This file may also be used under the terms of a commercial license
17
17
  # if purchased from OpenC3, Inc.
18
18
 
19
+ ENV['OPENC3_CLOUD'] ||= 'local'
20
+
19
21
  # Interface class implemented by each cloud provider: AWS, GCS, Azure
20
22
  module OpenC3
21
23
  class Bucket
@@ -51,7 +53,7 @@ module OpenC3
51
53
  raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
52
54
  end
53
55
 
54
- def list_objects(bucket:, prefix: nil)
56
+ def list_objects(bucket:, prefix: nil, max_request: nil, max_total: nil)
55
57
  raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
56
58
  end
57
59
 
@@ -17,7 +17,7 @@
17
17
  # All changes Copyright 2022, OpenC3, Inc.
18
18
  # All Rights Reserved
19
19
  #
20
- # This file may also be used under the terms of a commercial license
20
+ # This file may also be used under the terms of a commercial license
21
21
  # if purchased from OpenC3, Inc.
22
22
 
23
23
  require 'fileutils'
@@ -38,11 +38,8 @@ class BucketFile
38
38
  attr_reader :init_time
39
39
  attr_accessor :priority
40
40
 
41
- def initialize(bucket_path, client = nil)
42
- @bucket = client
43
- unless @bucket
44
- @bucket = OpenC3::Bucket.getClient()
45
- end
41
+ def initialize(bucket_path)
42
+ @bucket = OpenC3::Bucket.getClient()
46
43
  @bucket_path = bucket_path
47
44
  @local_path = nil
48
45
  @reservation_count = 0
@@ -149,8 +146,6 @@ class BucketFileCache
149
146
  end
150
147
 
151
148
  def initialize
152
- @bucket = OpenC3::Bucket.getClient()
153
-
154
149
  # Create local file cache location
155
150
  @cache_dir = Dir.mktmpdir
156
151
  FileUtils.mkdir_p(@cache_dir)
@@ -17,7 +17,7 @@
17
17
  # All changes Copyright 2022, OpenC3, Inc.
18
18
  # All Rights Reserved
19
19
  #
20
- # This file may also be used under the terms of a commercial license
20
+ # This file may also be used under the terms of a commercial license
21
21
  # if purchased from OpenC3, Inc.
22
22
 
23
23
  require 'openc3/utilities/bucket'
@@ -26,8 +26,19 @@ require 'zlib'
26
26
 
27
27
  module OpenC3
28
28
  class BucketUtilities
29
- def self.list_files_before_time(bucket, prefix, time)
30
- client = Bucket.getClient
29
+ FILE_TIMESTAMP_FORMAT = "%Y%m%d%H%M%S%N"
30
+ DIRECTORY_TIMESTAMP_FORMAT = "%Y%m%d"
31
+
32
+ # @param bucket [String] Name of the bucket to list
33
+ # @param prefix [String] Prefix to filter all files by
34
+ # @param start_time [Time|nil] Ruby time to find files after. nil means no start (first file on).
35
+ # @param end_time [Time|nil] Ruby time to find files before. nil means no end (up to last file).
36
+ # @param overlap [Boolean] Whether to include files which overlap the start and end time
37
+ # @param max_request [Integer] How many files to request in each API call
38
+ # @param max_total [Integer] Total number of files before stopping API requests
39
+ def self.files_between_time(bucket, prefix, start_time, end_time, file_suffix: nil,
40
+ overlap: false, max_request: 1000, max_total: 100_000)
41
+ client = Bucket.getClient()
31
42
  oldest_list = []
32
43
 
33
44
  # Return nothing if bucket doesn't exist (it won't at the very beginning)
@@ -35,23 +46,19 @@ module OpenC3
35
46
  return oldest_list
36
47
  end
37
48
 
38
- next_folder = false
39
- resp = client.list_objects(bucket: bucket, prefix: prefix)
40
- resp.each do |item|
41
- t = File.basename(item.key).split('__')[1]
42
- file_end_time = Time.utc(t[0..3], t[4..5], t[6..7], t[8..9], t[10..11], t[12..13])
43
- if file_end_time < time
44
- oldest_list << item.key
45
- else
46
- break
47
- end
49
+ directories = client.list_directories(bucket: bucket, path: prefix)
50
+ filtered_directories = filter_directories_to_time_range(directories, start_time, end_time)
51
+ filtered_directories.each do |directory|
52
+ directory_files = client.list_objects(bucket: bucket, prefix: "#{prefix}/#{directory}", max_request: max_request, max_total: max_total)
53
+ files = filter_files_to_time_range(directory_files, start_time, end_time, file_suffix: file_suffix, overlap: overlap)
54
+ oldest_list.concat(files)
48
55
  end
49
56
  return oldest_list
50
57
  end
51
58
 
52
59
  def self.move_log_file_to_bucket(filename, bucket_key, metadata: {})
53
60
  Thread.new do
54
- client = Bucket.getClient
61
+ client = Bucket.getClient()
55
62
 
56
63
  zipped = compress_file(filename)
57
64
  bucket_key = bucket_key + '.gz'
@@ -73,7 +80,7 @@ module OpenC3
73
80
  has_version_number = /(-|_|\.)\d+(-|_|\.)\d+(-|_|\.)\d+\./.match(filename)
74
81
  has_content_hash = /\.[a-f0-9]{20}\./.match(filename)
75
82
  return nil if has_version_number or has_content_hash
76
- return 'no-cache'
83
+ return 'no-store'
77
84
  end
78
85
 
79
86
  def self.compress_file(filename, chunk_size = 50_000_000)
@@ -105,5 +112,60 @@ module OpenC3
105
112
 
106
113
  return unzipped
107
114
  end
115
+
116
+ # Private methods
117
+
118
+ def self.filter_directories_to_time_range(directories, start_time, end_time)
119
+ result = []
120
+ directories.each do |directory|
121
+ result << directory if directory_in_time_range(directory, start_time, end_time)
122
+ end
123
+ return result
124
+ end
125
+
126
+ def self.directory_in_time_range(directory, start_time, end_time)
127
+ basename = File.basename(directory)
128
+ directory_start_time = DateTime.strptime(basename, DIRECTORY_TIMESTAMP_FORMAT).to_time
129
+ directory_end_time = directory_start_time + Time::SEC_PER_DAY
130
+ if (not start_time or start_time < directory_end_time) and (not end_time or end_time >= directory_start_time)
131
+ return true
132
+ else
133
+ return false
134
+ end
135
+ end
136
+
137
+ def self.filter_files_to_time_range(files, start_time, end_time, file_suffix: nil, overlap: false)
138
+ result = []
139
+ files.each do |file|
140
+ file_key = file.key.to_s
141
+ next if file_suffix and not file_key.end_with?(file_suffix)
142
+ if file_in_time_range(file_key, start_time, end_time, overlap: overlap)
143
+ result << file_key
144
+ end
145
+ end
146
+ return result
147
+ end
148
+
149
+ def self.file_in_time_range(bucket_path, start_time, end_time, overlap:)
150
+ file_start_time, file_end_time = get_file_times(bucket_path)
151
+ if overlap
152
+ if (not start_time or start_time <= file_end_time) and (not end_time or end_time >= file_start_time)
153
+ return true
154
+ end
155
+ else
156
+ if (not start_time or start_time <= file_start_time) and (not end_time or end_time >= file_end_time)
157
+ return true
158
+ end
159
+ end
160
+ return false
161
+ end
162
+
163
+ def self.get_file_times(bucket_path)
164
+ basename = File.basename(bucket_path)
165
+ file_start_timestamp, file_end_timestamp, other = basename.split("__")
166
+ file_start_time = DateTime.strptime(file_start_timestamp, FILE_TIMESTAMP_FORMAT).to_time
167
+ file_end_time = DateTime.strptime(file_end_timestamp, FILE_TIMESTAMP_FORMAT).to_time
168
+ return file_start_time, file_end_time
169
+ end
108
170
  end
109
171
  end
@@ -13,7 +13,7 @@
13
13
  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
14
  # GNU Affero General Public License for more details.
15
15
  #
16
- # This file may also be used under the terms of a commercial license
16
+ # This file may also be used under the terms of a commercial license
17
17
  # if purchased from OpenC3, Inc.
18
18
 
19
19
  require 'fileutils'
@@ -201,8 +201,9 @@ module OpenC3
201
201
  if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
202
202
  variables = plugin_hash['variables']
203
203
  if variables
204
- variables.delete("target_name")
205
- variables.delete("microservice_name")
204
+ PluginModel::RESERVED_VARIABLE_NAMES.each do |name|
205
+ variables.delete(name)
206
+ end
206
207
  end
207
208
  if plugin_file_path =~ Regexp.new("^#{OPENC3_LOCAL_MODE_PATH}/#{scope}/")
208
209
  # From local init - Always just update the exact one
@@ -400,14 +401,16 @@ module OpenC3
400
401
  files << split_key[2..-1].join('/') if include_temp
401
402
  next
402
403
  end
403
- found = false
404
- path_matchers.each do |path|
405
- if split_key.include?(path)
406
- found = true
407
- break
404
+ if path_matchers
405
+ found = false
406
+ path_matchers.each do |path|
407
+ if split_key.include?(path)
408
+ found = true
409
+ break
410
+ end
408
411
  end
412
+ next unless found
409
413
  end
410
- next unless found
411
414
  files << split_key[2..-1].join('/')
412
415
  end
413
416
  return files.sort
@@ -17,7 +17,7 @@
17
17
  # All changes Copyright 2022, OpenC3, Inc.
18
18
  # All Rights Reserved
19
19
  #
20
- # This file may also be used under the terms of a commercial license
20
+ # This file may also be used under the terms of a commercial license
21
21
  # if purchased from OpenC3, Inc.
22
22
 
23
23
  require 'openc3/core_ext/class'
@@ -41,11 +41,8 @@ module OpenC3
41
41
  # @return [String] Additional detail to add to messages
42
42
  instance_attr_accessor :detail_string
43
43
 
44
- # @return [String] Fluent tag
45
- instance_attr_accessor :tag
46
-
47
44
  # @return [String] Microservice name
48
- instance_attr_accessor :microservice_name
45
+ attr_reader :microservice_name
49
46
 
50
47
  # @return [String] Scope
51
48
  instance_attr_accessor :scope
@@ -78,11 +75,22 @@ module OpenC3
78
75
  @detail_string = nil
79
76
  @container_name = Socket.gethostname
80
77
  @microservice_name = nil
81
- @tag = @container_name + ".log"
82
- @mutex = Mutex.new
83
78
  @no_store = ENV['OPENC3_NO_STORE']
84
79
  end
85
80
 
81
+ # Only set the microservice name once (to help with multi microservices)
82
+ def microservice_name=(name)
83
+ @microservice_name = name unless @microservice_name
84
+ end
85
+
86
+ def self.microservice_name
87
+ self.instance.microservice_name
88
+ end
89
+
90
+ def self.microservice_name=(name)
91
+ self.instance.microservice_name = name
92
+ end
93
+
86
94
  # @param message [String] The message to print if the log level is at or
87
95
  # below the method name log level.
88
96
  # @param block [Proc] Block to call which should return a string to append
@@ -164,11 +172,11 @@ module OpenC3
164
172
  protected
165
173
 
166
174
  def log_message(severity_string, message, scope:, user:)
167
- @mutex.synchronize do
175
+ @@mutex.synchronize do
168
176
  data = { time: Time.now.to_nsec_from_epoch, '@timestamp' => Time.now.xmlschema(3), severity: severity_string }
169
177
  data[:microservice_name] = @microservice_name if @microservice_name
170
178
  data[:detail] = @detail_string if @detail_string
171
- data[:user] = user['name'] || 'Unknown' if user # EE: If a user is passed, put its name ('Unknown' if it doesn't have a name). Don't include user data if no user was passed
179
+ data[:user] = user['username'] || 'Unknown' if user # EE: If a user is passed, put its name ('Unknown' if it doesn't have a name). Don't include user data if no user was passed
172
180
  if block_given?
173
181
  message = yield
174
182
  end
@@ -17,7 +17,7 @@
17
17
  # All changes Copyright 2022, OpenC3, Inc.
18
18
  # All Rights Reserved
19
19
  #
20
- # This file may also be used under the terms of a commercial license
20
+ # This file may also be used under the terms of a commercial license
21
21
  # if purchased from OpenC3, Inc.
22
22
 
23
23
  require 'openc3/config/config_parser'
@@ -36,9 +36,10 @@ module OpenC3
36
36
  # @param tool_name [String] The name of the tool creating the message log.
37
37
  # This will be inserted into the message log filename to help identify it.
38
38
  # @param log_dir [String] The filesystem path to store the message log file.
39
- def initialize(tool_name, log_dir, scope:)
39
+ # @param tags [Array<String>] Array of strings to put into the filename
40
+ def initialize(tool_name, log_dir, tags: ['messages'], scope:)
40
41
  @remote_log_directory = "#{scope}/tool_logs/#{tool_name}/"
41
- @tool_name = tool_name
42
+ @tags = tags.unshift(tool_name)
42
43
  @log_dir = log_dir
43
44
  @filename = ''
44
45
  @file = nil
@@ -82,9 +83,9 @@ module OpenC3
82
83
  def start(take_mutex = true)
83
84
  @mutex.lock if take_mutex
84
85
  # Prevent starting files too fast
85
- sleep(0.1) until !File.exist?(File.join(@log_dir, File.build_timestamped_filename([@tool_name, 'messages'])))
86
+ sleep(0.1) until !File.exist?(File.join(@log_dir, File.build_timestamped_filename(@tags)))
86
87
  stop(false)
87
- timed_filename = File.build_timestamped_filename([@tool_name, 'messages'])
88
+ timed_filename = File.build_timestamped_filename(@tags)
88
89
  @start_day = timed_filename[0..9].gsub("_", "") # YYYYMMDD
89
90
  @filename = File.join(@log_dir, timed_filename)
90
91
  @file = File.open(@filename, 'a')
@@ -0,0 +1,22 @@
1
+ # encoding: ascii-8bit
2
+
3
+ # Copyright 2022 OpenC3, Inc.
4
+ # All Rights Reserved.
5
+ #
6
+ # This program is free software; you can modify and/or redistribute it
7
+ # under the terms of the GNU Affero General Public License
8
+ # as published by the Free Software Foundation; version 3 with
9
+ # attribution addendums as found in the LICENSE.txt
10
+ #
11
+ # This program is distributed in the hope that it will be useful,
12
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
+ # GNU Affero General Public License for more details.
15
+ #
16
+ # This file may also be used under the terms of a commercial license
17
+ # if purchased from OpenC3, Inc.
18
+
19
+ module OpenC3
20
+ class Migration
21
+ end
22
+ end
@@ -80,7 +80,6 @@ module OpenC3
80
80
  @redis_key = ENV['OPENC3_REDIS_PASSWORD']
81
81
  @redis_url = "redis://#{ENV['OPENC3_REDIS_HOSTNAME']}:#{ENV['OPENC3_REDIS_PORT']}"
82
82
  @redis_pool = ConnectionPool.new(size: pool_size) { build_redis() }
83
- @topic_offsets = {}
84
83
  end
85
84
 
86
85
  unless $openc3_redis_cluster
@@ -143,14 +142,16 @@ module OpenC3
143
142
  topics.each do |topic|
144
143
  # Normally we will just be grabbing the topic offset
145
144
  # this allows xread to get everything past this point
146
- last_id = @topic_offsets[topic]
145
+ Thread.current[:topic_offsets] ||= {}
146
+ topic_offsets = Thread.current[:topic_offsets]
147
+ last_id = topic_offsets[topic]
147
148
  if last_id
148
149
  offsets << last_id
149
150
  else
150
151
  # If there is no topic offset this is the first call.
151
152
  # Get the last offset ID so we'll start getting everything from now on
152
153
  offsets << get_last_offset(topic)
153
- @topic_offsets[topic] = offsets[-1]
154
+ topic_offsets[topic] = offsets[-1]
154
155
  end
155
156
  end
156
157
  return offsets
@@ -158,6 +159,8 @@ module OpenC3
158
159
 
159
160
  unless $openc3_redis_cluster
160
161
  def read_topics(topics, offsets = nil, timeout_ms = 1000, count = nil)
162
+ Thread.current[:topic_offsets] ||= {}
163
+ topic_offsets = Thread.current[:topic_offsets]
161
164
  begin
162
165
  # Logger.debug "read_topics: #{topics}, #{offsets} pool:#{@redis_pool}"
163
166
  @redis_pool.with do |redis|
@@ -166,7 +169,7 @@ module OpenC3
166
169
  if result and result.length > 0
167
170
  result.each do |topic, messages|
168
171
  messages.each do |msg_id, msg_hash|
169
- @topic_offsets[topic] = msg_id
172
+ topic_offsets[topic] = msg_id
170
173
  yield topic, msg_id, msg_hash, redis if block_given?
171
174
  end
172
175
  end
@@ -199,7 +202,6 @@ module OpenC3
199
202
  # @return [String] the entry id
200
203
  def write_topic(topic, msg_hash, id = '*', maxlen = nil, approximate = 'true')
201
204
  id = '*' if id.nil?
202
- # Logger.debug "write_topic topic:#{topic} id:#{id} hash:#{msg_hash}"
203
205
  @redis_pool.with do |redis|
204
206
  return redis.xadd(topic, msg_hash, id: id, maxlen: maxlen, approximate: approximate)
205
207
  end
@@ -13,7 +13,7 @@
13
13
  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
14
  # GNU Affero General Public License for more details.
15
15
  #
16
- # This file may also be used under the terms of a commercial license
16
+ # This file may also be used under the terms of a commercial license
17
17
  # if purchased from OpenC3, Inc.
18
18
 
19
19
  require 'fileutils'
@@ -44,14 +44,16 @@ module OpenC3
44
44
  next
45
45
  end
46
46
 
47
- found = false
48
- path_matchers.each do |path|
49
- if split_key.include?(path)
50
- found = true
51
- break
47
+ if path_matchers
48
+ found = false
49
+ path_matchers.each do |path|
50
+ if split_key.include?(path)
51
+ found = true
52
+ break
53
+ end
52
54
  end
55
+ next unless found
53
56
  end
54
- next unless found
55
57
  result_no_scope_or_target_folder = split_key[2..-1].join('/')
56
58
  if object.key.include?("#{scope}/targets_modified")
57
59
  modified << result_no_scope_or_target_folder
@@ -1,14 +1,14 @@
1
1
  # encoding: ascii-8bit
2
2
 
3
- OPENC3_VERSION = '5.1.1'
3
+ OPENC3_VERSION = '5.2.0'
4
4
  module OpenC3
5
5
  module Version
6
6
  MAJOR = '5'
7
- MINOR = '1'
8
- PATCH = '1'
7
+ MINOR = '2'
8
+ PATCH = '0'
9
9
  OTHER = ''
10
- BUILD = 'c94738fcddee73959df8a60ac8746ab38a16c17c'
10
+ BUILD = '49c62512b75f45a8cbfc20a6fb5ed114bab35e23'
11
11
  end
12
- VERSION = '5.1.1'
13
- GEM_VERSION = '5.1.1'
12
+ VERSION = '5.2.0'
13
+ GEM_VERSION = '5.2.0'
14
14
  end
data/lib/openc3.rb CHANGED
@@ -17,7 +17,7 @@
17
17
  # All changes Copyright 2022, OpenC3, Inc.
18
18
  # All Rights Reserved
19
19
  #
20
- # This file may also be used under the terms of a commercial license
20
+ # This file may also be used under the terms of a commercial license
21
21
  # if purchased from OpenC3, Inc.
22
22
 
23
23
  # This file sets up using the OpenC3 framework
@@ -51,3 +51,4 @@ require 'openc3/system'
51
51
  # OpenC3 services need to die if something goes wrong so they can be restarted
52
52
  require 'thread'
53
53
  Thread.abort_on_exception = true
54
+ Thread.report_on_exception = true
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: openc3
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.1.1
4
+ version: 5.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ryan Melton
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2022-11-19 00:00:00.000000000 Z
12
+ date: 2022-12-13 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: bundler
@@ -711,14 +711,17 @@ files:
711
711
  - lib/openc3/accessors/html_accessor.rb
712
712
  - lib/openc3/accessors/json_accessor.rb
713
713
  - lib/openc3/accessors/xml_accessor.rb
714
+ - lib/openc3/api/README.md
714
715
  - lib/openc3/api/api.rb
715
716
  - lib/openc3/api/authorized_api.rb
716
717
  - lib/openc3/api/cmd_api.rb
717
718
  - lib/openc3/api/config_api.rb
718
719
  - lib/openc3/api/interface_api.rb
719
720
  - lib/openc3/api/limits_api.rb
721
+ - lib/openc3/api/offline_access_api.rb
720
722
  - lib/openc3/api/router_api.rb
721
723
  - lib/openc3/api/settings_api.rb
724
+ - lib/openc3/api/stash_api.rb
722
725
  - lib/openc3/api/target_api.rb
723
726
  - lib/openc3/api/tlm_api.rb
724
727
  - lib/openc3/bridge/bridge.rb
@@ -810,6 +813,8 @@ files:
810
813
  - lib/openc3/microservices/interface_microservice.rb
811
814
  - lib/openc3/microservices/log_microservice.rb
812
815
  - lib/openc3/microservices/microservice.rb
816
+ - lib/openc3/microservices/multi_microservice.rb
817
+ - lib/openc3/microservices/periodic_microservice.rb
813
818
  - lib/openc3/microservices/plugin_microservice.rb
814
819
  - lib/openc3/microservices/reaction_microservice.rb
815
820
  - lib/openc3/microservices/reducer_microservice.rb
@@ -817,6 +822,8 @@ files:
817
822
  - lib/openc3/microservices/text_log_microservice.rb
818
823
  - lib/openc3/microservices/timeline_microservice.rb
819
824
  - lib/openc3/microservices/trigger_group_microservice.rb
825
+ - lib/openc3/migrations/20221202214600_add_target_names.rb
826
+ - lib/openc3/migrations/20221210174900_convert_to_multi.rb
820
827
  - lib/openc3/models/activity_model.rb
821
828
  - lib/openc3/models/auth_model.rb
822
829
  - lib/openc3/models/cvt_model.rb
@@ -829,9 +836,11 @@ files:
829
836
  - lib/openc3/models/metric_model.rb
830
837
  - lib/openc3/models/microservice_model.rb
831
838
  - lib/openc3/models/microservice_status_model.rb
839
+ - lib/openc3/models/migration_model.rb
832
840
  - lib/openc3/models/model.rb
833
841
  - lib/openc3/models/note_model.rb
834
842
  - lib/openc3/models/notification_model.rb
843
+ - lib/openc3/models/offline_access_model.rb
835
844
  - lib/openc3/models/ping_model.rb
836
845
  - lib/openc3/models/plugin_model.rb
837
846
  - lib/openc3/models/process_status_model.rb
@@ -842,6 +851,7 @@ files:
842
851
  - lib/openc3/models/scope_model.rb
843
852
  - lib/openc3/models/settings_model.rb
844
853
  - lib/openc3/models/sorted_model.rb
854
+ - lib/openc3/models/stash_model.rb
845
855
  - lib/openc3/models/target_model.rb
846
856
  - lib/openc3/models/timeline_model.rb
847
857
  - lib/openc3/models/tool_config_model.rb
@@ -878,11 +888,11 @@ files:
878
888
  - lib/openc3/processors/watermark_processor.rb
879
889
  - lib/openc3/script.rb
880
890
  - lib/openc3/script/api_shared.rb
881
- - lib/openc3/script/calendar.rb
882
891
  - lib/openc3/script/commands.rb
883
892
  - lib/openc3/script/exceptions.rb
884
893
  - lib/openc3/script/extract.rb
885
894
  - lib/openc3/script/limits.rb
895
+ - lib/openc3/script/metadata.rb
886
896
  - lib/openc3/script/script.rb
887
897
  - lib/openc3/script/script_runner.rb
888
898
  - lib/openc3/script/storage.rb
@@ -937,6 +947,7 @@ files:
937
947
  - lib/openc3/utilities/logger.rb
938
948
  - lib/openc3/utilities/message_log.rb
939
949
  - lib/openc3/utilities/metric.rb
950
+ - lib/openc3/utilities/migration.rb
940
951
  - lib/openc3/utilities/open_telemetry.rb
941
952
  - lib/openc3/utilities/process_manager.rb
942
953
  - lib/openc3/utilities/quaternion.rb