openc3 5.1.1 → 5.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/bin/openc3cli +48 -9
  3. data/data/config/interface_modifiers.yaml +14 -0
  4. data/data/config/parameter_modifiers.yaml +5 -3
  5. data/data/config/screen.yaml +12 -8
  6. data/data/config/target.yaml +33 -0
  7. data/ext/openc3/ext/config_parser/config_parser.c +66 -63
  8. data/ext/openc3/ext/packet/packet.c +1 -4
  9. data/lib/openc3/api/README.md +5 -0
  10. data/lib/openc3/api/api.rb +3 -1
  11. data/lib/openc3/api/cmd_api.rb +43 -112
  12. data/lib/openc3/api/interface_api.rb +3 -3
  13. data/lib/openc3/api/offline_access_api.rb +78 -0
  14. data/lib/openc3/api/settings_api.rb +3 -1
  15. data/lib/openc3/api/stash_api.rb +63 -0
  16. data/lib/openc3/api/target_api.rb +4 -5
  17. data/lib/openc3/config/config_parser.rb +47 -47
  18. data/lib/openc3/interfaces/interface.rb +11 -1
  19. data/lib/openc3/interfaces/protocols/burst_protocol.rb +30 -16
  20. data/lib/openc3/interfaces/protocols/fixed_protocol.rb +8 -2
  21. data/lib/openc3/interfaces/protocols/ignore_packet_protocol.rb +2 -2
  22. data/lib/openc3/interfaces/protocols/override_protocol.rb +2 -2
  23. data/lib/openc3/interfaces/tcpip_server_interface.rb +3 -1
  24. data/lib/openc3/io/json_api_object.rb +30 -9
  25. data/lib/openc3/io/json_drb.rb +6 -1
  26. data/lib/openc3/io/json_drb_object.rb +18 -9
  27. data/lib/openc3/io/json_rpc.rb +5 -3
  28. data/lib/openc3/logs/buffered_packet_log_writer.rb +1 -1
  29. data/lib/openc3/logs/log_writer.rb +8 -2
  30. data/lib/openc3/microservices/cleanup_microservice.rb +3 -3
  31. data/lib/openc3/microservices/decom_microservice.rb +8 -8
  32. data/lib/openc3/microservices/interface_microservice.rb +86 -71
  33. data/lib/openc3/microservices/log_microservice.rb +5 -3
  34. data/lib/openc3/microservices/microservice.rb +18 -14
  35. data/lib/openc3/microservices/multi_microservice.rb +62 -0
  36. data/lib/openc3/microservices/periodic_microservice.rb +58 -0
  37. data/lib/openc3/microservices/reaction_microservice.rb +61 -47
  38. data/lib/openc3/microservices/reducer_microservice.rb +64 -40
  39. data/lib/openc3/microservices/router_microservice.rb +4 -4
  40. data/lib/openc3/microservices/text_log_microservice.rb +2 -2
  41. data/lib/openc3/microservices/timeline_microservice.rb +44 -30
  42. data/lib/openc3/microservices/trigger_group_microservice.rb +39 -36
  43. data/lib/openc3/migrations/20221202214600_add_target_names.rb +30 -0
  44. data/lib/openc3/migrations/20221210174900_convert_to_multi.rb +65 -0
  45. data/lib/openc3/models/cvt_model.rb +1 -1
  46. data/lib/openc3/models/gem_model.rb +24 -20
  47. data/lib/openc3/models/interface_model.rb +69 -35
  48. data/lib/openc3/models/metadata_model.rb +1 -1
  49. data/lib/openc3/models/microservice_model.rb +7 -24
  50. data/lib/openc3/models/migration_model.rb +52 -0
  51. data/lib/openc3/models/model.rb +2 -7
  52. data/lib/openc3/models/note_model.rb +1 -1
  53. data/lib/openc3/models/offline_access_model.rb +55 -0
  54. data/lib/openc3/models/plugin_model.rb +12 -3
  55. data/lib/openc3/models/reaction_model.rb +6 -2
  56. data/lib/openc3/models/scope_model.rb +89 -13
  57. data/lib/openc3/models/settings_model.rb +1 -1
  58. data/lib/openc3/models/stash_model.rb +53 -0
  59. data/lib/openc3/models/target_model.rb +301 -130
  60. data/lib/openc3/models/tool_model.rb +1 -12
  61. data/lib/openc3/models/widget_model.rb +1 -6
  62. data/lib/openc3/operators/microservice_operator.rb +45 -6
  63. data/lib/openc3/operators/operator.rb +27 -5
  64. data/lib/openc3/packets/commands.rb +1 -25
  65. data/lib/openc3/packets/limits.rb +0 -75
  66. data/lib/openc3/packets/packet.rb +0 -28
  67. data/lib/openc3/packets/packet_item.rb +23 -0
  68. data/lib/openc3/packets/packet_item_limits.rb +2 -2
  69. data/lib/openc3/packets/parsers/state_parser.rb +10 -6
  70. data/lib/openc3/packets/telemetry.rb +1 -45
  71. data/lib/openc3/script/commands.rb +41 -71
  72. data/lib/openc3/script/extract.rb +15 -1
  73. data/lib/openc3/script/{calendar.rb → metadata.rb} +42 -17
  74. data/lib/openc3/script/script.rb +13 -5
  75. data/lib/openc3/script/storage.rb +3 -1
  76. data/lib/openc3/system/system.rb +19 -17
  77. data/lib/openc3/tools/cmd_tlm_server/interface_thread.rb +4 -4
  78. data/lib/openc3/top_level.rb +3 -3
  79. data/lib/openc3/topics/command_decom_topic.rb +2 -2
  80. data/lib/openc3/topics/command_topic.rb +7 -6
  81. data/lib/openc3/topics/interface_topic.rb +2 -2
  82. data/lib/openc3/topics/router_topic.rb +1 -1
  83. data/lib/openc3/topics/telemetry_topic.rb +2 -1
  84. data/lib/openc3/utilities/authentication.rb +35 -14
  85. data/lib/openc3/utilities/aws_bucket.rb +4 -3
  86. data/lib/openc3/utilities/bucket.rb +4 -2
  87. data/lib/openc3/utilities/bucket_file_cache.rb +3 -8
  88. data/lib/openc3/utilities/bucket_utilities.rb +77 -15
  89. data/lib/openc3/utilities/local_mode.rb +12 -9
  90. data/lib/openc3/utilities/logger.rb +17 -9
  91. data/lib/openc3/utilities/message_log.rb +6 -5
  92. data/lib/openc3/utilities/migration.rb +22 -0
  93. data/lib/openc3/utilities/store_autoload.rb +7 -5
  94. data/lib/openc3/utilities/target_file.rb +9 -7
  95. data/lib/openc3/version.rb +6 -6
  96. data/lib/openc3.rb +2 -1
  97. metadata +14 -3
@@ -17,7 +17,7 @@
17
17
  # All changes Copyright 2022, OpenC3, Inc.
18
18
  # All Rights Reserved
19
19
  #
20
- # This file may also be used under the terms of a commercial license
20
+ # This file may also be used under the terms of a commercial license
21
21
  # if purchased from OpenC3, Inc.
22
22
 
23
23
  require 'openc3/version'
@@ -60,6 +60,8 @@ module OpenC3
60
60
 
61
61
  REFRESH_OFFSET_SECONDS = 60
62
62
 
63
+ attr_reader :refresh_token
64
+
63
65
  # @param url [String] The url of the openc3 or keycloak in the cluster
64
66
  def initialize(url)
65
67
  @url = url
@@ -72,7 +74,7 @@ module OpenC3
72
74
  end
73
75
 
74
76
  # Load the token from the environment
75
- def token()
77
+ def token
76
78
  @auth_mutex.synchronize do
77
79
  @log = [nil, nil]
78
80
  current_time = Time.now.to_i
@@ -87,23 +89,42 @@ module OpenC3
87
89
  "Bearer #{@token}"
88
90
  end
89
91
 
92
+ def get_token_from_refresh_token(refresh_token)
93
+ current_time = Time.now.to_i
94
+ begin
95
+ @refresh_token = refresh_token
96
+ _refresh_token(current_time)
97
+ return @token
98
+ rescue OpenC3AuthenticationError
99
+ return nil
100
+ end
101
+ return nil
102
+ end
103
+
90
104
  private
91
105
 
92
106
  # Make the token and save token to instance
93
107
  def _make_token(current_time)
94
108
  client_id = ENV['OPENC3_API_CLIENT'] || 'api'
95
- data = "username=#{ENV['OPENC3_API_USER']}&password=#{ENV['OPENC3_API_PASSWORD']}"
96
- data << "&client_id=#{client_id}"
97
- data << '&grant_type=password&scope=openid'
98
- headers = {
99
- 'Content-Type' => 'application/x-www-form-urlencoded',
100
- 'User-Agent' => "OpenC3KeycloakAuthorization / #{OPENC3_VERSION} (ruby/openc3/lib/utilities/authentication)",
101
- }
102
- oath = _make_request(headers, data)
103
- @token = oath['access_token']
104
- @refresh_token = oath['refresh_token']
105
- @expires_at = current_time + oath['expires_in'] - REFRESH_OFFSET_SECONDS
106
- @refresh_expires_at = current_time + oath['refresh_expires_in'] - REFRESH_OFFSET_SECONDS
109
+ if ENV['OPENC3_API_USER'] and ENV['OPENC3_API_PASSWORD']
110
+ # Username and password
111
+ data = "username=#{ENV['OPENC3_API_USER']}&password=#{ENV['OPENC3_API_PASSWORD']}"
112
+ data << "&client_id=#{client_id}"
113
+ data << '&grant_type=password&scope=openid'
114
+ headers = {
115
+ 'Content-Type' => 'application/x-www-form-urlencoded',
116
+ 'User-Agent' => "OpenC3KeycloakAuthorization / #{OPENC3_VERSION} (ruby/openc3/lib/utilities/authentication)",
117
+ }
118
+ oath = _make_request(headers, data)
119
+ @token = oath['access_token']
120
+ @refresh_token = oath['refresh_token']
121
+ @expires_at = current_time + oath['expires_in'] - REFRESH_OFFSET_SECONDS
122
+ @refresh_expires_at = current_time + oath['refresh_expires_in'] - REFRESH_OFFSET_SECONDS
123
+ else
124
+ # Offline Access Token
125
+ @refresh_token ||= ENV['OPENC3_API_TOKEN']
126
+ _refresh_token(current_time)
127
+ end
107
128
  end
108
129
 
109
130
  # Refresh the token and save token to instance
@@ -13,7 +13,7 @@
13
13
  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
14
  # GNU Affero General Public License for more details.
15
15
  #
16
- # This file may also be used under the terms of a commercial license
16
+ # This file may also be used under the terms of a commercial license
17
17
  # if purchased from OpenC3, Inc.
18
18
 
19
19
  require 'openc3/utilities/bucket'
@@ -107,12 +107,13 @@ module OpenC3
107
107
  nil
108
108
  end
109
109
 
110
- def list_objects(bucket:, prefix: nil)
110
+ def list_objects(bucket:, prefix: nil, max_request: 1000, max_total: 100_000)
111
111
  token = nil
112
112
  result = []
113
113
  while true
114
- resp = @client.list_objects_v2(bucket: bucket, prefix: prefix, max_keys: 1000)
114
+ resp = @client.list_objects_v2(bucket: bucket, prefix: prefix, max_keys: max_request)
115
115
  result.concat(resp.contents)
116
+ break if result.length >= max_total
116
117
  break unless resp.is_truncated
117
118
  token = resp.next_continuation_token
118
119
  end
@@ -13,9 +13,11 @@
13
13
  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
14
  # GNU Affero General Public License for more details.
15
15
  #
16
- # This file may also be used under the terms of a commercial license
16
+ # This file may also be used under the terms of a commercial license
17
17
  # if purchased from OpenC3, Inc.
18
18
 
19
+ ENV['OPENC3_CLOUD'] ||= 'local'
20
+
19
21
  # Interface class implemented by each cloud provider: AWS, GCS, Azure
20
22
  module OpenC3
21
23
  class Bucket
@@ -51,7 +53,7 @@ module OpenC3
51
53
  raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
52
54
  end
53
55
 
54
- def list_objects(bucket:, prefix: nil)
56
+ def list_objects(bucket:, prefix: nil, max_request: nil, max_total: nil)
55
57
  raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
56
58
  end
57
59
 
@@ -17,7 +17,7 @@
17
17
  # All changes Copyright 2022, OpenC3, Inc.
18
18
  # All Rights Reserved
19
19
  #
20
- # This file may also be used under the terms of a commercial license
20
+ # This file may also be used under the terms of a commercial license
21
21
  # if purchased from OpenC3, Inc.
22
22
 
23
23
  require 'fileutils'
@@ -38,11 +38,8 @@ class BucketFile
38
38
  attr_reader :init_time
39
39
  attr_accessor :priority
40
40
 
41
- def initialize(bucket_path, client = nil)
42
- @bucket = client
43
- unless @bucket
44
- @bucket = OpenC3::Bucket.getClient()
45
- end
41
+ def initialize(bucket_path)
42
+ @bucket = OpenC3::Bucket.getClient()
46
43
  @bucket_path = bucket_path
47
44
  @local_path = nil
48
45
  @reservation_count = 0
@@ -149,8 +146,6 @@ class BucketFileCache
149
146
  end
150
147
 
151
148
  def initialize
152
- @bucket = OpenC3::Bucket.getClient()
153
-
154
149
  # Create local file cache location
155
150
  @cache_dir = Dir.mktmpdir
156
151
  FileUtils.mkdir_p(@cache_dir)
@@ -17,7 +17,7 @@
17
17
  # All changes Copyright 2022, OpenC3, Inc.
18
18
  # All Rights Reserved
19
19
  #
20
- # This file may also be used under the terms of a commercial license
20
+ # This file may also be used under the terms of a commercial license
21
21
  # if purchased from OpenC3, Inc.
22
22
 
23
23
  require 'openc3/utilities/bucket'
@@ -26,8 +26,19 @@ require 'zlib'
26
26
 
27
27
  module OpenC3
28
28
  class BucketUtilities
29
- def self.list_files_before_time(bucket, prefix, time)
30
- client = Bucket.getClient
29
+ FILE_TIMESTAMP_FORMAT = "%Y%m%d%H%M%S%N"
30
+ DIRECTORY_TIMESTAMP_FORMAT = "%Y%m%d"
31
+
32
+ # @param bucket [String] Name of the bucket to list
33
+ # @param prefix [String] Prefix to filter all files by
34
+ # @param start_time [Time|nil] Ruby time to find files after. nil means no start (first file on).
35
+ # @param end_time [Time|nil] Ruby time to find files before. nil means no end (up to last file).
36
+ # @param overlap [Boolean] Whether to include files which overlap the start and end time
37
+ # @param max_request [Integer] How many files to request in each API call
38
+ # @param max_total [Integer] Total number of files before stopping API requests
39
+ def self.files_between_time(bucket, prefix, start_time, end_time, file_suffix: nil,
40
+ overlap: false, max_request: 1000, max_total: 100_000)
41
+ client = Bucket.getClient()
31
42
  oldest_list = []
32
43
 
33
44
  # Return nothing if bucket doesn't exist (it won't at the very beginning)
@@ -35,23 +46,19 @@ module OpenC3
35
46
  return oldest_list
36
47
  end
37
48
 
38
- next_folder = false
39
- resp = client.list_objects(bucket: bucket, prefix: prefix)
40
- resp.each do |item|
41
- t = File.basename(item.key).split('__')[1]
42
- file_end_time = Time.utc(t[0..3], t[4..5], t[6..7], t[8..9], t[10..11], t[12..13])
43
- if file_end_time < time
44
- oldest_list << item.key
45
- else
46
- break
47
- end
49
+ directories = client.list_directories(bucket: bucket, path: prefix)
50
+ filtered_directories = filter_directories_to_time_range(directories, start_time, end_time)
51
+ filtered_directories.each do |directory|
52
+ directory_files = client.list_objects(bucket: bucket, prefix: "#{prefix}/#{directory}", max_request: max_request, max_total: max_total)
53
+ files = filter_files_to_time_range(directory_files, start_time, end_time, file_suffix: file_suffix, overlap: overlap)
54
+ oldest_list.concat(files)
48
55
  end
49
56
  return oldest_list
50
57
  end
51
58
 
52
59
  def self.move_log_file_to_bucket(filename, bucket_key, metadata: {})
53
60
  Thread.new do
54
- client = Bucket.getClient
61
+ client = Bucket.getClient()
55
62
 
56
63
  zipped = compress_file(filename)
57
64
  bucket_key = bucket_key + '.gz'
@@ -73,7 +80,7 @@ module OpenC3
73
80
  has_version_number = /(-|_|\.)\d+(-|_|\.)\d+(-|_|\.)\d+\./.match(filename)
74
81
  has_content_hash = /\.[a-f0-9]{20}\./.match(filename)
75
82
  return nil if has_version_number or has_content_hash
76
- return 'no-cache'
83
+ return 'no-store'
77
84
  end
78
85
 
79
86
  def self.compress_file(filename, chunk_size = 50_000_000)
@@ -105,5 +112,60 @@ module OpenC3
105
112
 
106
113
  return unzipped
107
114
  end
115
+
116
+ # Private methods
117
+
118
+ def self.filter_directories_to_time_range(directories, start_time, end_time)
119
+ result = []
120
+ directories.each do |directory|
121
+ result << directory if directory_in_time_range(directory, start_time, end_time)
122
+ end
123
+ return result
124
+ end
125
+
126
+ def self.directory_in_time_range(directory, start_time, end_time)
127
+ basename = File.basename(directory)
128
+ directory_start_time = DateTime.strptime(basename, DIRECTORY_TIMESTAMP_FORMAT).to_time
129
+ directory_end_time = directory_start_time + Time::SEC_PER_DAY
130
+ if (not start_time or start_time < directory_end_time) and (not end_time or end_time >= directory_start_time)
131
+ return true
132
+ else
133
+ return false
134
+ end
135
+ end
136
+
137
+ def self.filter_files_to_time_range(files, start_time, end_time, file_suffix: nil, overlap: false)
138
+ result = []
139
+ files.each do |file|
140
+ file_key = file.key.to_s
141
+ next if file_suffix and not file_key.end_with?(file_suffix)
142
+ if file_in_time_range(file_key, start_time, end_time, overlap: overlap)
143
+ result << file_key
144
+ end
145
+ end
146
+ return result
147
+ end
148
+
149
+ def self.file_in_time_range(bucket_path, start_time, end_time, overlap:)
150
+ file_start_time, file_end_time = get_file_times(bucket_path)
151
+ if overlap
152
+ if (not start_time or start_time <= file_end_time) and (not end_time or end_time >= file_start_time)
153
+ return true
154
+ end
155
+ else
156
+ if (not start_time or start_time <= file_start_time) and (not end_time or end_time >= file_end_time)
157
+ return true
158
+ end
159
+ end
160
+ return false
161
+ end
162
+
163
+ def self.get_file_times(bucket_path)
164
+ basename = File.basename(bucket_path)
165
+ file_start_timestamp, file_end_timestamp, other = basename.split("__")
166
+ file_start_time = DateTime.strptime(file_start_timestamp, FILE_TIMESTAMP_FORMAT).to_time
167
+ file_end_time = DateTime.strptime(file_end_timestamp, FILE_TIMESTAMP_FORMAT).to_time
168
+ return file_start_time, file_end_time
169
+ end
108
170
  end
109
171
  end
@@ -13,7 +13,7 @@
13
13
  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
14
  # GNU Affero General Public License for more details.
15
15
  #
16
- # This file may also be used under the terms of a commercial license
16
+ # This file may also be used under the terms of a commercial license
17
17
  # if purchased from OpenC3, Inc.
18
18
 
19
19
  require 'fileutils'
@@ -201,8 +201,9 @@ module OpenC3
201
201
  if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
202
202
  variables = plugin_hash['variables']
203
203
  if variables
204
- variables.delete("target_name")
205
- variables.delete("microservice_name")
204
+ PluginModel::RESERVED_VARIABLE_NAMES.each do |name|
205
+ variables.delete(name)
206
+ end
206
207
  end
207
208
  if plugin_file_path =~ Regexp.new("^#{OPENC3_LOCAL_MODE_PATH}/#{scope}/")
208
209
  # From local init - Always just update the exact one
@@ -400,14 +401,16 @@ module OpenC3
400
401
  files << split_key[2..-1].join('/') if include_temp
401
402
  next
402
403
  end
403
- found = false
404
- path_matchers.each do |path|
405
- if split_key.include?(path)
406
- found = true
407
- break
404
+ if path_matchers
405
+ found = false
406
+ path_matchers.each do |path|
407
+ if split_key.include?(path)
408
+ found = true
409
+ break
410
+ end
408
411
  end
412
+ next unless found
409
413
  end
410
- next unless found
411
414
  files << split_key[2..-1].join('/')
412
415
  end
413
416
  return files.sort
@@ -17,7 +17,7 @@
17
17
  # All changes Copyright 2022, OpenC3, Inc.
18
18
  # All Rights Reserved
19
19
  #
20
- # This file may also be used under the terms of a commercial license
20
+ # This file may also be used under the terms of a commercial license
21
21
  # if purchased from OpenC3, Inc.
22
22
 
23
23
  require 'openc3/core_ext/class'
@@ -41,11 +41,8 @@ module OpenC3
41
41
  # @return [String] Additional detail to add to messages
42
42
  instance_attr_accessor :detail_string
43
43
 
44
- # @return [String] Fluent tag
45
- instance_attr_accessor :tag
46
-
47
44
  # @return [String] Microservice name
48
- instance_attr_accessor :microservice_name
45
+ attr_reader :microservice_name
49
46
 
50
47
  # @return [String] Scope
51
48
  instance_attr_accessor :scope
@@ -78,11 +75,22 @@ module OpenC3
78
75
  @detail_string = nil
79
76
  @container_name = Socket.gethostname
80
77
  @microservice_name = nil
81
- @tag = @container_name + ".log"
82
- @mutex = Mutex.new
83
78
  @no_store = ENV['OPENC3_NO_STORE']
84
79
  end
85
80
 
81
+ # Only set the microservice name once (to help with multi microservices)
82
+ def microservice_name=(name)
83
+ @microservice_name = name unless @microservice_name
84
+ end
85
+
86
+ def self.microservice_name
87
+ self.instance.microservice_name
88
+ end
89
+
90
+ def self.microservice_name=(name)
91
+ self.instance.microservice_name = name
92
+ end
93
+
86
94
  # @param message [String] The message to print if the log level is at or
87
95
  # below the method name log level.
88
96
  # @param block [Proc] Block to call which should return a string to append
@@ -164,11 +172,11 @@ module OpenC3
164
172
  protected
165
173
 
166
174
  def log_message(severity_string, message, scope:, user:)
167
- @mutex.synchronize do
175
+ @@mutex.synchronize do
168
176
  data = { time: Time.now.to_nsec_from_epoch, '@timestamp' => Time.now.xmlschema(3), severity: severity_string }
169
177
  data[:microservice_name] = @microservice_name if @microservice_name
170
178
  data[:detail] = @detail_string if @detail_string
171
- data[:user] = user['name'] || 'Unknown' if user # EE: If a user is passed, put its name ('Unknown' if it doesn't have a name). Don't include user data if no user was passed
179
+ data[:user] = user['username'] || 'Unknown' if user # EE: If a user is passed, put its name ('Unknown' if it doesn't have a name). Don't include user data if no user was passed
172
180
  if block_given?
173
181
  message = yield
174
182
  end
@@ -17,7 +17,7 @@
17
17
  # All changes Copyright 2022, OpenC3, Inc.
18
18
  # All Rights Reserved
19
19
  #
20
- # This file may also be used under the terms of a commercial license
20
+ # This file may also be used under the terms of a commercial license
21
21
  # if purchased from OpenC3, Inc.
22
22
 
23
23
  require 'openc3/config/config_parser'
@@ -36,9 +36,10 @@ module OpenC3
36
36
  # @param tool_name [String] The name of the tool creating the message log.
37
37
  # This will be inserted into the message log filename to help identify it.
38
38
  # @param log_dir [String] The filesystem path to store the message log file.
39
- def initialize(tool_name, log_dir, scope:)
39
+ # @param tags [Array<String>] Array of strings to put into the filename
40
+ def initialize(tool_name, log_dir, tags: ['messages'], scope:)
40
41
  @remote_log_directory = "#{scope}/tool_logs/#{tool_name}/"
41
- @tool_name = tool_name
42
+ @tags = tags.unshift(tool_name)
42
43
  @log_dir = log_dir
43
44
  @filename = ''
44
45
  @file = nil
@@ -82,9 +83,9 @@ module OpenC3
82
83
  def start(take_mutex = true)
83
84
  @mutex.lock if take_mutex
84
85
  # Prevent starting files too fast
85
- sleep(0.1) until !File.exist?(File.join(@log_dir, File.build_timestamped_filename([@tool_name, 'messages'])))
86
+ sleep(0.1) until !File.exist?(File.join(@log_dir, File.build_timestamped_filename(@tags)))
86
87
  stop(false)
87
- timed_filename = File.build_timestamped_filename([@tool_name, 'messages'])
88
+ timed_filename = File.build_timestamped_filename(@tags)
88
89
  @start_day = timed_filename[0..9].gsub("_", "") # YYYYMMDD
89
90
  @filename = File.join(@log_dir, timed_filename)
90
91
  @file = File.open(@filename, 'a')
@@ -0,0 +1,22 @@
1
+ # encoding: ascii-8bit
2
+
3
+ # Copyright 2022 OpenC3, Inc.
4
+ # All Rights Reserved.
5
+ #
6
+ # This program is free software; you can modify and/or redistribute it
7
+ # under the terms of the GNU Affero General Public License
8
+ # as published by the Free Software Foundation; version 3 with
9
+ # attribution addendums as found in the LICENSE.txt
10
+ #
11
+ # This program is distributed in the hope that it will be useful,
12
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
+ # GNU Affero General Public License for more details.
15
+ #
16
+ # This file may also be used under the terms of a commercial license
17
+ # if purchased from OpenC3, Inc.
18
+
19
+ module OpenC3
20
+ class Migration
21
+ end
22
+ end
@@ -80,7 +80,6 @@ module OpenC3
80
80
  @redis_key = ENV['OPENC3_REDIS_PASSWORD']
81
81
  @redis_url = "redis://#{ENV['OPENC3_REDIS_HOSTNAME']}:#{ENV['OPENC3_REDIS_PORT']}"
82
82
  @redis_pool = ConnectionPool.new(size: pool_size) { build_redis() }
83
- @topic_offsets = {}
84
83
  end
85
84
 
86
85
  unless $openc3_redis_cluster
@@ -143,14 +142,16 @@ module OpenC3
143
142
  topics.each do |topic|
144
143
  # Normally we will just be grabbing the topic offset
145
144
  # this allows xread to get everything past this point
146
- last_id = @topic_offsets[topic]
145
+ Thread.current[:topic_offsets] ||= {}
146
+ topic_offsets = Thread.current[:topic_offsets]
147
+ last_id = topic_offsets[topic]
147
148
  if last_id
148
149
  offsets << last_id
149
150
  else
150
151
  # If there is no topic offset this is the first call.
151
152
  # Get the last offset ID so we'll start getting everything from now on
152
153
  offsets << get_last_offset(topic)
153
- @topic_offsets[topic] = offsets[-1]
154
+ topic_offsets[topic] = offsets[-1]
154
155
  end
155
156
  end
156
157
  return offsets
@@ -158,6 +159,8 @@ module OpenC3
158
159
 
159
160
  unless $openc3_redis_cluster
160
161
  def read_topics(topics, offsets = nil, timeout_ms = 1000, count = nil)
162
+ Thread.current[:topic_offsets] ||= {}
163
+ topic_offsets = Thread.current[:topic_offsets]
161
164
  begin
162
165
  # Logger.debug "read_topics: #{topics}, #{offsets} pool:#{@redis_pool}"
163
166
  @redis_pool.with do |redis|
@@ -166,7 +169,7 @@ module OpenC3
166
169
  if result and result.length > 0
167
170
  result.each do |topic, messages|
168
171
  messages.each do |msg_id, msg_hash|
169
- @topic_offsets[topic] = msg_id
172
+ topic_offsets[topic] = msg_id
170
173
  yield topic, msg_id, msg_hash, redis if block_given?
171
174
  end
172
175
  end
@@ -199,7 +202,6 @@ module OpenC3
199
202
  # @return [String] the entry id
200
203
  def write_topic(topic, msg_hash, id = '*', maxlen = nil, approximate = 'true')
201
204
  id = '*' if id.nil?
202
- # Logger.debug "write_topic topic:#{topic} id:#{id} hash:#{msg_hash}"
203
205
  @redis_pool.with do |redis|
204
206
  return redis.xadd(topic, msg_hash, id: id, maxlen: maxlen, approximate: approximate)
205
207
  end
@@ -13,7 +13,7 @@
13
13
  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
14
  # GNU Affero General Public License for more details.
15
15
  #
16
- # This file may also be used under the terms of a commercial license
16
+ # This file may also be used under the terms of a commercial license
17
17
  # if purchased from OpenC3, Inc.
18
18
 
19
19
  require 'fileutils'
@@ -44,14 +44,16 @@ module OpenC3
44
44
  next
45
45
  end
46
46
 
47
- found = false
48
- path_matchers.each do |path|
49
- if split_key.include?(path)
50
- found = true
51
- break
47
+ if path_matchers
48
+ found = false
49
+ path_matchers.each do |path|
50
+ if split_key.include?(path)
51
+ found = true
52
+ break
53
+ end
52
54
  end
55
+ next unless found
53
56
  end
54
- next unless found
55
57
  result_no_scope_or_target_folder = split_key[2..-1].join('/')
56
58
  if object.key.include?("#{scope}/targets_modified")
57
59
  modified << result_no_scope_or_target_folder
@@ -1,14 +1,14 @@
1
1
  # encoding: ascii-8bit
2
2
 
3
- OPENC3_VERSION = '5.1.1'
3
+ OPENC3_VERSION = '5.2.0'
4
4
  module OpenC3
5
5
  module Version
6
6
  MAJOR = '5'
7
- MINOR = '1'
8
- PATCH = '1'
7
+ MINOR = '2'
8
+ PATCH = '0'
9
9
  OTHER = ''
10
- BUILD = 'c94738fcddee73959df8a60ac8746ab38a16c17c'
10
+ BUILD = '49c62512b75f45a8cbfc20a6fb5ed114bab35e23'
11
11
  end
12
- VERSION = '5.1.1'
13
- GEM_VERSION = '5.1.1'
12
+ VERSION = '5.2.0'
13
+ GEM_VERSION = '5.2.0'
14
14
  end
data/lib/openc3.rb CHANGED
@@ -17,7 +17,7 @@
17
17
  # All changes Copyright 2022, OpenC3, Inc.
18
18
  # All Rights Reserved
19
19
  #
20
- # This file may also be used under the terms of a commercial license
20
+ # This file may also be used under the terms of a commercial license
21
21
  # if purchased from OpenC3, Inc.
22
22
 
23
23
  # This file sets up using the OpenC3 framework
@@ -51,3 +51,4 @@ require 'openc3/system'
51
51
  # OpenC3 services need to die if something goes wrong so they can be restarted
52
52
  require 'thread'
53
53
  Thread.abort_on_exception = true
54
+ Thread.report_on_exception = true
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: openc3
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.1.1
4
+ version: 5.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ryan Melton
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2022-11-19 00:00:00.000000000 Z
12
+ date: 2022-12-13 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: bundler
@@ -711,14 +711,17 @@ files:
711
711
  - lib/openc3/accessors/html_accessor.rb
712
712
  - lib/openc3/accessors/json_accessor.rb
713
713
  - lib/openc3/accessors/xml_accessor.rb
714
+ - lib/openc3/api/README.md
714
715
  - lib/openc3/api/api.rb
715
716
  - lib/openc3/api/authorized_api.rb
716
717
  - lib/openc3/api/cmd_api.rb
717
718
  - lib/openc3/api/config_api.rb
718
719
  - lib/openc3/api/interface_api.rb
719
720
  - lib/openc3/api/limits_api.rb
721
+ - lib/openc3/api/offline_access_api.rb
720
722
  - lib/openc3/api/router_api.rb
721
723
  - lib/openc3/api/settings_api.rb
724
+ - lib/openc3/api/stash_api.rb
722
725
  - lib/openc3/api/target_api.rb
723
726
  - lib/openc3/api/tlm_api.rb
724
727
  - lib/openc3/bridge/bridge.rb
@@ -810,6 +813,8 @@ files:
810
813
  - lib/openc3/microservices/interface_microservice.rb
811
814
  - lib/openc3/microservices/log_microservice.rb
812
815
  - lib/openc3/microservices/microservice.rb
816
+ - lib/openc3/microservices/multi_microservice.rb
817
+ - lib/openc3/microservices/periodic_microservice.rb
813
818
  - lib/openc3/microservices/plugin_microservice.rb
814
819
  - lib/openc3/microservices/reaction_microservice.rb
815
820
  - lib/openc3/microservices/reducer_microservice.rb
@@ -817,6 +822,8 @@ files:
817
822
  - lib/openc3/microservices/text_log_microservice.rb
818
823
  - lib/openc3/microservices/timeline_microservice.rb
819
824
  - lib/openc3/microservices/trigger_group_microservice.rb
825
+ - lib/openc3/migrations/20221202214600_add_target_names.rb
826
+ - lib/openc3/migrations/20221210174900_convert_to_multi.rb
820
827
  - lib/openc3/models/activity_model.rb
821
828
  - lib/openc3/models/auth_model.rb
822
829
  - lib/openc3/models/cvt_model.rb
@@ -829,9 +836,11 @@ files:
829
836
  - lib/openc3/models/metric_model.rb
830
837
  - lib/openc3/models/microservice_model.rb
831
838
  - lib/openc3/models/microservice_status_model.rb
839
+ - lib/openc3/models/migration_model.rb
832
840
  - lib/openc3/models/model.rb
833
841
  - lib/openc3/models/note_model.rb
834
842
  - lib/openc3/models/notification_model.rb
843
+ - lib/openc3/models/offline_access_model.rb
835
844
  - lib/openc3/models/ping_model.rb
836
845
  - lib/openc3/models/plugin_model.rb
837
846
  - lib/openc3/models/process_status_model.rb
@@ -842,6 +851,7 @@ files:
842
851
  - lib/openc3/models/scope_model.rb
843
852
  - lib/openc3/models/settings_model.rb
844
853
  - lib/openc3/models/sorted_model.rb
854
+ - lib/openc3/models/stash_model.rb
845
855
  - lib/openc3/models/target_model.rb
846
856
  - lib/openc3/models/timeline_model.rb
847
857
  - lib/openc3/models/tool_config_model.rb
@@ -878,11 +888,11 @@ files:
878
888
  - lib/openc3/processors/watermark_processor.rb
879
889
  - lib/openc3/script.rb
880
890
  - lib/openc3/script/api_shared.rb
881
- - lib/openc3/script/calendar.rb
882
891
  - lib/openc3/script/commands.rb
883
892
  - lib/openc3/script/exceptions.rb
884
893
  - lib/openc3/script/extract.rb
885
894
  - lib/openc3/script/limits.rb
895
+ - lib/openc3/script/metadata.rb
886
896
  - lib/openc3/script/script.rb
887
897
  - lib/openc3/script/script_runner.rb
888
898
  - lib/openc3/script/storage.rb
@@ -937,6 +947,7 @@ files:
937
947
  - lib/openc3/utilities/logger.rb
938
948
  - lib/openc3/utilities/message_log.rb
939
949
  - lib/openc3/utilities/metric.rb
950
+ - lib/openc3/utilities/migration.rb
940
951
  - lib/openc3/utilities/open_telemetry.rb
941
952
  - lib/openc3/utilities/process_manager.rb
942
953
  - lib/openc3/utilities/quaternion.rb