forest_admin_datasource_rpc 1.19.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/forest_admin_datasource_rpc/Utils/rpc_client.rb +0 -4
- data/lib/forest_admin_datasource_rpc/Utils/schema_polling_client.rb +76 -96
- data/lib/forest_admin_datasource_rpc/datasource.rb +6 -32
- data/lib/forest_admin_datasource_rpc/version.rb +1 -1
- data/lib/forest_admin_datasource_rpc.rb +97 -42
- metadata +2 -3
- data/lib/forest_admin_datasource_rpc/Utils/schema_polling_pool.rb +0 -286
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f0b6bdcda842a6d936560f74f955186c51f36d4be1dbd332c51d0ecdea710cf5
|
|
4
|
+
data.tar.gz: faabc27ad8e0b3b9fe5f719da93375224c8e4f42bec99dff71c9a132c58119fc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9fe4da7443c4080e0097ad823e750a4cc5fab2e992a1890cc22568ed055cc0b7f78702ca4523bcc0ecb56715355d02c89915d4bffbfd80e9e861ee128633cbe2
|
|
7
|
+
data.tar.gz: ff37fcf51cf77f521b284dc4f21358241a1c5cd5859117cf614b507a1f4cb0836f547cf0220222052b0cc05a5f898debeb1ab3bf01aa182767fca3a3ff769543
|
|
@@ -32,8 +32,6 @@ module ForestAdminDatasourceRpc
|
|
|
32
32
|
|
|
33
33
|
HTTP_NOT_MODIFIED = 304
|
|
34
34
|
NotModified = Class.new
|
|
35
|
-
DEFAULT_TIMEOUT = 30 # seconds
|
|
36
|
-
DEFAULT_OPEN_TIMEOUT = 10 # seconds
|
|
37
35
|
|
|
38
36
|
def initialize(api_url, auth_secret)
|
|
39
37
|
@api_url = api_url
|
|
@@ -65,8 +63,6 @@ module ForestAdminDatasourceRpc
|
|
|
65
63
|
faraday.response :json, parser_options: { symbolize_names: symbolize_keys }
|
|
66
64
|
faraday.adapter Faraday.default_adapter
|
|
67
65
|
faraday.ssl.verify = !ForestAdminAgent::Facades::Container.cache(:debug)
|
|
68
|
-
faraday.options.timeout = DEFAULT_TIMEOUT
|
|
69
|
-
faraday.options.open_timeout = DEFAULT_OPEN_TIMEOUT
|
|
70
66
|
end
|
|
71
67
|
|
|
72
68
|
timestamp = Time.now.utc.iso8601(3)
|
|
@@ -1,50 +1,54 @@
|
|
|
1
1
|
require 'openssl'
|
|
2
2
|
require 'json'
|
|
3
3
|
require 'time'
|
|
4
|
-
require 'digest'
|
|
5
4
|
|
|
6
5
|
module ForestAdminDatasourceRpc
|
|
7
6
|
module Utils
|
|
8
7
|
class SchemaPollingClient
|
|
9
|
-
attr_reader :closed
|
|
8
|
+
attr_reader :closed
|
|
10
9
|
|
|
11
|
-
DEFAULT_POLLING_INTERVAL = 600
|
|
12
|
-
MIN_POLLING_INTERVAL = 1
|
|
13
|
-
MAX_POLLING_INTERVAL = 3600
|
|
10
|
+
DEFAULT_POLLING_INTERVAL = 600 # seconds (10 minutes)
|
|
11
|
+
MIN_POLLING_INTERVAL = 1 # seconds (minimum safe interval)
|
|
12
|
+
MAX_POLLING_INTERVAL = 3600 # seconds (1 hour max)
|
|
14
13
|
|
|
15
|
-
def initialize(uri, auth_secret,
|
|
16
|
-
&on_schema_change)
|
|
14
|
+
def initialize(uri, auth_secret, options = {}, &on_schema_change)
|
|
17
15
|
@uri = uri
|
|
18
16
|
@auth_secret = auth_secret
|
|
19
|
-
@polling_interval = polling_interval
|
|
17
|
+
@polling_interval = options[:polling_interval] || DEFAULT_POLLING_INTERVAL
|
|
20
18
|
@on_schema_change = on_schema_change
|
|
21
19
|
@closed = false
|
|
22
|
-
@introspection_schema = introspection_schema
|
|
23
|
-
@current_schema = nil
|
|
24
20
|
@cached_etag = nil
|
|
21
|
+
@polling_thread = nil
|
|
22
|
+
@mutex = Mutex.new
|
|
25
23
|
@connection_attempts = 0
|
|
26
|
-
@initial_sync_completed = false
|
|
27
|
-
@client_id = uri
|
|
28
24
|
|
|
25
|
+
# Validate polling interval
|
|
29
26
|
validate_polling_interval!
|
|
30
27
|
|
|
28
|
+
# RPC client for schema fetching with ETag support
|
|
31
29
|
@rpc_client = RpcClient.new(@uri, @auth_secret)
|
|
32
30
|
end
|
|
33
31
|
|
|
34
|
-
def start
|
|
35
|
-
return
|
|
36
|
-
|
|
37
|
-
ForestAdminAgent::Facades::Container.logger&.log('Info', "Getting schema from RPC agent on #{@uri}.")
|
|
38
|
-
fetch_initial_schema_sync
|
|
32
|
+
def start
|
|
33
|
+
return if @closed
|
|
39
34
|
|
|
40
|
-
|
|
41
|
-
|
|
35
|
+
@mutex.synchronize do
|
|
36
|
+
return if @polling_thread&.alive?
|
|
37
|
+
|
|
38
|
+
@polling_thread = Thread.new do
|
|
39
|
+
polling_loop
|
|
40
|
+
rescue StandardError => e
|
|
41
|
+
ForestAdminAgent::Facades::Container.logger&.log(
|
|
42
|
+
'Error',
|
|
43
|
+
"[Schema Polling] Unexpected error in polling loop: #{e.class} - #{e.message}"
|
|
44
|
+
)
|
|
45
|
+
end
|
|
46
|
+
end
|
|
42
47
|
|
|
43
48
|
ForestAdminAgent::Facades::Container.logger&.log(
|
|
44
49
|
'Info',
|
|
45
|
-
"[Schema Polling]
|
|
50
|
+
"[Schema Polling] Polling started (interval: #{@polling_interval}s)"
|
|
46
51
|
)
|
|
47
|
-
true
|
|
48
52
|
end
|
|
49
53
|
|
|
50
54
|
def stop
|
|
@@ -53,11 +57,46 @@ module ForestAdminDatasourceRpc
|
|
|
53
57
|
@closed = true
|
|
54
58
|
ForestAdminAgent::Facades::Container.logger&.log('Debug', '[Schema Polling] Stopping polling')
|
|
55
59
|
|
|
56
|
-
|
|
60
|
+
@mutex.synchronize do
|
|
61
|
+
if @polling_thread&.alive?
|
|
62
|
+
@polling_thread.kill
|
|
63
|
+
@polling_thread = nil
|
|
64
|
+
end
|
|
65
|
+
end
|
|
57
66
|
|
|
58
67
|
ForestAdminAgent::Facades::Container.logger&.log('Debug', '[Schema Polling] Polling stopped')
|
|
59
68
|
end
|
|
60
69
|
|
|
70
|
+
private
|
|
71
|
+
|
|
72
|
+
def polling_loop
|
|
73
|
+
ForestAdminAgent::Facades::Container.logger&.log(
|
|
74
|
+
'Debug',
|
|
75
|
+
"[Schema Polling] Starting polling loop (interval: #{@polling_interval}s)"
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
loop do
|
|
79
|
+
break if @closed
|
|
80
|
+
|
|
81
|
+
begin
|
|
82
|
+
check_schema
|
|
83
|
+
rescue StandardError => e
|
|
84
|
+
handle_error(e)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Sleep with interrupt check (check every second for early termination)
|
|
88
|
+
ForestAdminAgent::Facades::Container.logger&.log(
|
|
89
|
+
'Debug',
|
|
90
|
+
"[Schema Polling] Waiting #{@polling_interval}s before next check (current ETag: #{@cached_etag || "none"})"
|
|
91
|
+
)
|
|
92
|
+
remaining = @polling_interval
|
|
93
|
+
while remaining.positive? && !@closed
|
|
94
|
+
sleep([remaining, 1].min)
|
|
95
|
+
remaining -= 1
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
61
100
|
def check_schema
|
|
62
101
|
@connection_attempts += 1
|
|
63
102
|
log_checking_schema
|
|
@@ -72,65 +111,11 @@ module ForestAdminDatasourceRpc
|
|
|
72
111
|
log_unexpected_error(e)
|
|
73
112
|
end
|
|
74
113
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
Digest::SHA1.hexdigest(schema.to_json)
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
def fetch_initial_schema_sync
|
|
84
|
-
# If we have an introspection schema, send its ETag to avoid re-downloading unchanged schema
|
|
85
|
-
introspection_etag = compute_etag(@introspection_schema) if @introspection_schema
|
|
86
|
-
result = @rpc_client.fetch_schema('/forest/rpc-schema', if_none_match: introspection_etag)
|
|
87
|
-
|
|
88
|
-
if result == RpcClient::NotModified
|
|
89
|
-
# Schema unchanged from introspection - use introspection
|
|
90
|
-
@current_schema = @introspection_schema
|
|
91
|
-
@cached_etag = introspection_etag
|
|
92
|
-
@initial_sync_completed = true
|
|
93
|
-
ForestAdminAgent::Facades::Container.logger&.log(
|
|
94
|
-
'Info',
|
|
95
|
-
"[Schema Polling] RPC schema unchanged (HTTP 304), using introspection (ETag: #{@cached_etag})"
|
|
96
|
-
)
|
|
97
|
-
else
|
|
98
|
-
# New schema from RPC
|
|
99
|
-
@current_schema = result.body
|
|
100
|
-
@cached_etag = result.etag || compute_etag(@current_schema)
|
|
101
|
-
@initial_sync_completed = true
|
|
102
|
-
ForestAdminAgent::Facades::Container.logger&.log(
|
|
103
|
-
'Debug',
|
|
104
|
-
"[Schema Polling] Initial schema fetched successfully (ETag: #{@cached_etag})"
|
|
105
|
-
)
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
@introspection_schema = nil
|
|
109
|
-
rescue Faraday::ConnectionFailed, Faraday::TimeoutError,
|
|
110
|
-
ForestAdminAgent::Http::Exceptions::AuthenticationOpenIdClient, StandardError => e
|
|
111
|
-
handle_initial_fetch_error(e)
|
|
112
|
-
end
|
|
113
|
-
|
|
114
|
-
def handle_initial_fetch_error(error)
|
|
115
|
-
if @introspection_schema
|
|
116
|
-
# Fallback to introspection schema - don't crash
|
|
117
|
-
@current_schema = @introspection_schema
|
|
118
|
-
@cached_etag = compute_etag(@current_schema)
|
|
119
|
-
@introspection_schema = nil
|
|
120
|
-
@initial_sync_completed = true
|
|
121
|
-
ForestAdminAgent::Facades::Container.logger&.log(
|
|
122
|
-
'Warn',
|
|
123
|
-
"RPC agent at #{@uri} is unreachable (#{error.class}: #{error.message}), " \
|
|
124
|
-
"using provided introspection schema (ETag: #{@cached_etag})"
|
|
125
|
-
)
|
|
126
|
-
else
|
|
127
|
-
# No introspection - re-raise to crash
|
|
128
|
-
ForestAdminAgent::Facades::Container.logger&.log(
|
|
129
|
-
'Error',
|
|
130
|
-
"Failed to get schema from RPC agent at #{@uri}: #{error.class} - #{error.message}"
|
|
131
|
-
)
|
|
132
|
-
raise error
|
|
133
|
-
end
|
|
114
|
+
def handle_error(error)
|
|
115
|
+
ForestAdminAgent::Facades::Container.logger&.log(
|
|
116
|
+
'Error',
|
|
117
|
+
"[Schema Polling] Error during schema check: #{error.class} - #{error.message}"
|
|
118
|
+
)
|
|
134
119
|
end
|
|
135
120
|
|
|
136
121
|
def trigger_schema_change_callback(schema)
|
|
@@ -177,27 +162,22 @@ module ForestAdminDatasourceRpc
|
|
|
177
162
|
end
|
|
178
163
|
|
|
179
164
|
def handle_schema_changed(result)
|
|
180
|
-
|
|
181
|
-
new_etag
|
|
182
|
-
|
|
183
|
-
if @initial_sync_completed
|
|
184
|
-
handle_schema_update(new_schema, new_etag)
|
|
185
|
-
else
|
|
186
|
-
@cached_etag = new_etag
|
|
187
|
-
@current_schema = new_schema
|
|
188
|
-
@initial_sync_completed = true
|
|
189
|
-
ForestAdminAgent::Facades::Container.logger&.log(
|
|
190
|
-
'Info',
|
|
191
|
-
"[Schema Polling] Initial sync completed successfully (ETag: #{new_etag})"
|
|
192
|
-
)
|
|
193
|
-
end
|
|
165
|
+
new_etag = result.etag
|
|
166
|
+
@cached_etag.nil? ? handle_initial_schema(new_etag) : handle_schema_update(result.body, new_etag)
|
|
194
167
|
@connection_attempts = 0
|
|
195
168
|
end
|
|
196
169
|
|
|
170
|
+
def handle_initial_schema(etag)
|
|
171
|
+
@cached_etag = etag
|
|
172
|
+
ForestAdminAgent::Facades::Container.logger&.log(
|
|
173
|
+
'Debug',
|
|
174
|
+
"[Schema Polling] Initial schema loaded successfully (ETag: #{etag})"
|
|
175
|
+
)
|
|
176
|
+
end
|
|
177
|
+
|
|
197
178
|
def handle_schema_update(schema, etag)
|
|
198
179
|
old_etag = @cached_etag
|
|
199
180
|
@cached_etag = etag
|
|
200
|
-
@current_schema = schema
|
|
201
181
|
msg = "[Schema Polling] Schema changed detected (old ETag: #{old_etag}, new ETag: #{etag}), " \
|
|
202
182
|
'triggering reload callback'
|
|
203
183
|
ForestAdminAgent::Facades::Container.logger&.log('Info', msg)
|
|
@@ -25,9 +25,6 @@ module ForestAdminDatasourceRpc
|
|
|
25
25
|
@live_query_connections = native_query_connections.to_h { |conn| [conn[:name], conn[:name]] }
|
|
26
26
|
|
|
27
27
|
@schema = { charts: @charts }
|
|
28
|
-
|
|
29
|
-
# Register shutdown hook to cleanup schema polling gracefully
|
|
30
|
-
register_shutdown_hook if @schema_polling_client
|
|
31
28
|
end
|
|
32
29
|
|
|
33
30
|
def render_chart(caller, name)
|
|
@@ -62,38 +59,15 @@ module ForestAdminDatasourceRpc
|
|
|
62
59
|
@cleaned_up = true
|
|
63
60
|
|
|
64
61
|
if @schema_polling_client
|
|
65
|
-
|
|
62
|
+
ForestAdminAgent::Facades::Container.logger&.log('Info', '[RPCDatasource] Stopping schema polling...')
|
|
66
63
|
@schema_polling_client.stop
|
|
67
|
-
|
|
64
|
+
ForestAdminAgent::Facades::Container.logger&.log('Info', '[RPCDatasource] Schema polling stopped')
|
|
68
65
|
end
|
|
69
66
|
rescue StandardError => e
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
def register_shutdown_hook
|
|
76
|
-
# Register at_exit hook for graceful shutdown
|
|
77
|
-
# This ensures schema polling is stopped when the application exits
|
|
78
|
-
at_exit do
|
|
79
|
-
cleanup
|
|
80
|
-
end
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
def log_info(message)
|
|
84
|
-
return unless defined?(ForestAdminAgent::Facades::Container)
|
|
85
|
-
|
|
86
|
-
ForestAdminAgent::Facades::Container.logger&.log('Info', message)
|
|
87
|
-
rescue StandardError
|
|
88
|
-
# Silently ignore logging errors during shutdown
|
|
89
|
-
end
|
|
90
|
-
|
|
91
|
-
def log_error(message)
|
|
92
|
-
return unless defined?(ForestAdminAgent::Facades::Container)
|
|
93
|
-
|
|
94
|
-
ForestAdminAgent::Facades::Container.logger&.log('Error', message)
|
|
95
|
-
rescue StandardError
|
|
96
|
-
# Silently ignore logging errors during shutdown
|
|
67
|
+
ForestAdminAgent::Facades::Container.logger&.log(
|
|
68
|
+
'Error',
|
|
69
|
+
"[RPCDatasource] Error during cleanup: #{e.class} - #{e.message}"
|
|
70
|
+
)
|
|
97
71
|
end
|
|
98
72
|
end
|
|
99
73
|
end
|
|
@@ -8,61 +8,116 @@ loader.setup
|
|
|
8
8
|
module ForestAdminDatasourceRpc
|
|
9
9
|
class Error < StandardError; end
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
11
|
+
# Build a RPC datasource with schema polling enabled.
|
|
12
|
+
#
|
|
13
|
+
# @param options [Hash] Configuration options
|
|
14
|
+
# @option options [String] :uri The URI of the RPC agent
|
|
15
|
+
# @option options [String] :auth_secret The authentication secret (optional, will use cache if not provided)
|
|
16
|
+
# @option options [Integer] :schema_polling_interval Polling interval in seconds (optional)
|
|
17
|
+
# - Default: 600 seconds (10 minutes)
|
|
18
|
+
# - Can be overridden with ENV['SCHEMA_POLLING_INTERVAL']
|
|
19
|
+
# - Valid range: 1-3600 seconds
|
|
20
|
+
# - Priority: options[:schema_polling_interval] > ENV['SCHEMA_POLLING_INTERVAL'] > default
|
|
21
|
+
# - Example: SCHEMA_POLLING_INTERVAL=30 for development (30 seconds)
|
|
22
|
+
#
|
|
23
|
+
# @return [ForestAdminDatasourceRpc::Datasource] The configured datasource with schema polling
|
|
15
24
|
def self.build(options)
|
|
16
25
|
uri = options[:uri]
|
|
17
26
|
auth_secret = options[:auth_secret] || ForestAdminAgent::Facades::Container.cache(:auth_secret)
|
|
18
|
-
|
|
27
|
+
ForestAdminAgent::Facades::Container.logger.log('Info', "Getting schema from RPC agent on #{uri}.")
|
|
19
28
|
|
|
20
|
-
|
|
21
|
-
ENV['SCHEMA_POLLING_INTERVAL_SEC']&.to_i ||
|
|
22
|
-
600
|
|
29
|
+
schema = nil
|
|
23
30
|
|
|
24
|
-
|
|
25
|
-
|
|
31
|
+
begin
|
|
32
|
+
rpc_client = Utils::RpcClient.new(uri, auth_secret)
|
|
33
|
+
response = rpc_client.fetch_schema('/forest/rpc-schema')
|
|
34
|
+
schema = response.body
|
|
35
|
+
rescue Faraday::ConnectionFailed => e
|
|
36
|
+
ForestAdminAgent::Facades::Container.logger.log(
|
|
37
|
+
'Error',
|
|
38
|
+
"Connection failed to RPC agent at #{uri}: #{e.message}\n#{e.backtrace.join("\n")}"
|
|
39
|
+
)
|
|
40
|
+
rescue Faraday::TimeoutError => e
|
|
41
|
+
ForestAdminAgent::Facades::Container.logger.log(
|
|
42
|
+
'Error',
|
|
43
|
+
"Request timeout to RPC agent at #{uri}: #{e.message}"
|
|
44
|
+
)
|
|
45
|
+
rescue ForestAdminAgent::Http::Exceptions::AuthenticationOpenIdClient => e
|
|
46
|
+
ForestAdminAgent::Facades::Container.logger.log(
|
|
47
|
+
'Error',
|
|
48
|
+
"Authentication failed with RPC agent at #{uri}: #{e.message}"
|
|
49
|
+
)
|
|
50
|
+
rescue StandardError => e
|
|
51
|
+
ForestAdminAgent::Facades::Container.logger.log(
|
|
52
|
+
'Error',
|
|
53
|
+
"Failed to get schema from RPC agent at #{uri}: #{e.class} - #{e.message}\n#{e.backtrace.join("\n")}"
|
|
54
|
+
)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
if schema.nil?
|
|
58
|
+
# return empty datasource for not breaking stack
|
|
59
|
+
ForestAdminDatasourceToolkit::Datasource.new
|
|
60
|
+
else
|
|
61
|
+
# Create schema polling client with configurable polling interval
|
|
62
|
+
# Priority: options[:schema_polling_interval] > ENV['SCHEMA_POLLING_INTERVAL'] > default (600)
|
|
63
|
+
polling_interval = if options[:schema_polling_interval]
|
|
64
|
+
options[:schema_polling_interval]
|
|
65
|
+
elsif ENV['SCHEMA_POLLING_INTERVAL']
|
|
66
|
+
ENV['SCHEMA_POLLING_INTERVAL'].to_i
|
|
67
|
+
else
|
|
68
|
+
600 # 10 minutes by default
|
|
69
|
+
end
|
|
26
70
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
Thread.new do
|
|
71
|
+
polling_options = {
|
|
72
|
+
polling_interval: polling_interval
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
schema_polling = Utils::SchemaPollingClient.new(uri, auth_secret, polling_options) do
|
|
76
|
+
# Callback when schema change is detected
|
|
34
77
|
logger = ForestAdminAgent::Facades::Container.logger
|
|
35
|
-
logger.log('Info', '[RPCDatasource] Schema change detected, reloading agent
|
|
36
|
-
|
|
37
|
-
ForestAdminAgent::Builder::AgentFactory.instance.reload!
|
|
38
|
-
logger.log('Info', '[RPCDatasource] Agent reload completed successfully')
|
|
39
|
-
rescue StandardError => e
|
|
40
|
-
logger.log('Error', "[RPCDatasource] Agent reload failed: #{e.class} - #{e.message}")
|
|
41
|
-
end
|
|
78
|
+
logger.log('Info', '[RPCDatasource] Schema change detected, reloading agent...')
|
|
79
|
+
ForestAdminAgent::Builder::AgentFactory.instance.reload!
|
|
42
80
|
end
|
|
43
|
-
|
|
81
|
+
schema_polling.start
|
|
44
82
|
|
|
45
|
-
|
|
46
|
-
# - Without introspection: crashes if RPC is unreachable
|
|
47
|
-
# - With introspection: falls back to introspection if RPC is unreachable
|
|
48
|
-
schema_polling.start?
|
|
83
|
+
datasource = ForestAdminDatasourceRpc::Datasource.new(options, schema, schema_polling)
|
|
49
84
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
raise ForestAdminDatasourceToolkit::Exceptions::ForestException,
|
|
53
|
-
'Fatal: Unable to build RPC datasource - no introspection schema was provided and schema fetch failed'
|
|
54
|
-
end
|
|
85
|
+
# Setup cleanup hooks for proper schema polling client shutdown
|
|
86
|
+
setup_cleanup_hooks(datasource)
|
|
55
87
|
|
|
56
|
-
|
|
88
|
+
datasource
|
|
89
|
+
end
|
|
57
90
|
end
|
|
58
91
|
|
|
59
|
-
def self.
|
|
60
|
-
|
|
61
|
-
|
|
92
|
+
def self.setup_cleanup_hooks(datasource)
|
|
93
|
+
# Register cleanup handler for graceful shutdown
|
|
94
|
+
at_exit do
|
|
95
|
+
datasource.cleanup
|
|
96
|
+
rescue StandardError => e
|
|
97
|
+
# Silently ignore errors during exit cleanup to prevent test pollution
|
|
98
|
+
warn "[RPCDatasource] Error during at_exit cleanup: #{e.message}" if $VERBOSE
|
|
99
|
+
end
|
|
62
100
|
|
|
63
|
-
#
|
|
64
|
-
|
|
65
|
-
|
|
101
|
+
# Handle SIGINT (Ctrl+C)
|
|
102
|
+
Signal.trap('INT') do
|
|
103
|
+
begin
|
|
104
|
+
ForestAdminAgent::Facades::Container.logger&.log('Info', '[RPCDatasource] Received SIGINT, cleaning up...')
|
|
105
|
+
rescue StandardError
|
|
106
|
+
# Logger might not be available
|
|
107
|
+
end
|
|
108
|
+
datasource.cleanup
|
|
109
|
+
exit(0)
|
|
110
|
+
end
|
|
66
111
|
|
|
67
|
-
|
|
112
|
+
# Handle SIGTERM (default kill signal)
|
|
113
|
+
Signal.trap('TERM') do
|
|
114
|
+
begin
|
|
115
|
+
ForestAdminAgent::Facades::Container.logger&.log('Info', '[RPCDatasource] Received SIGTERM, cleaning up...')
|
|
116
|
+
rescue StandardError
|
|
117
|
+
# Logger might not be available
|
|
118
|
+
end
|
|
119
|
+
datasource.cleanup
|
|
120
|
+
exit(0)
|
|
121
|
+
end
|
|
122
|
+
end
|
|
68
123
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: forest_admin_datasource_rpc
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 2.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Matthieu
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: exe
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2025-12-
|
|
12
|
+
date: 2025-12-11 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: base64
|
|
@@ -126,7 +126,6 @@ files:
|
|
|
126
126
|
- lib/forest_admin_datasource_rpc.rb
|
|
127
127
|
- lib/forest_admin_datasource_rpc/Utils/rpc_client.rb
|
|
128
128
|
- lib/forest_admin_datasource_rpc/Utils/schema_polling_client.rb
|
|
129
|
-
- lib/forest_admin_datasource_rpc/Utils/schema_polling_pool.rb
|
|
130
129
|
- lib/forest_admin_datasource_rpc/collection.rb
|
|
131
130
|
- lib/forest_admin_datasource_rpc/datasource.rb
|
|
132
131
|
- lib/forest_admin_datasource_rpc/version.rb
|
|
@@ -1,286 +0,0 @@
|
|
|
1
|
-
require 'singleton'
|
|
2
|
-
|
|
3
|
-
module ForestAdminDatasourceRpc
|
|
4
|
-
module Utils
|
|
5
|
-
# Thread pool manager for RPC schema polling.
|
|
6
|
-
# Uses a single scheduler thread that dispatches polling tasks to a bounded
|
|
7
|
-
# pool of worker threads, preventing thread exhaustion when many RPC slaves
|
|
8
|
-
# are configured.
|
|
9
|
-
#
|
|
10
|
-
# Design principles:
|
|
11
|
-
# - Minimal mutex hold times to avoid blocking HTTP request threads
|
|
12
|
-
# - Workers yield control frequently to prevent GIL starvation
|
|
13
|
-
# - Non-blocking queue operations where possible
|
|
14
|
-
class SchemaPollingPool
|
|
15
|
-
include Singleton
|
|
16
|
-
|
|
17
|
-
DEFAULT_MAX_THREADS = 5
|
|
18
|
-
MIN_THREADS = 1
|
|
19
|
-
MAX_THREADS = 50
|
|
20
|
-
SCHEDULER_INTERVAL = 1
|
|
21
|
-
INITIAL_STAGGER_WINDOW = 30
|
|
22
|
-
|
|
23
|
-
attr_reader :max_threads, :configured
|
|
24
|
-
|
|
25
|
-
def initialize
|
|
26
|
-
@mutex = Mutex.new
|
|
27
|
-
@clients = {}
|
|
28
|
-
@work_queue = Queue.new
|
|
29
|
-
@workers = []
|
|
30
|
-
@running = false
|
|
31
|
-
@max_threads = DEFAULT_MAX_THREADS
|
|
32
|
-
@shutdown_requested = false
|
|
33
|
-
@configured = false
|
|
34
|
-
@scheduler_thread = nil
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
# Configure the pool before starting. Must be called before any clients register.
|
|
38
|
-
# @param max_threads [Integer] Maximum number of worker threads (1-20)
|
|
39
|
-
def configure(max_threads:)
|
|
40
|
-
@mutex.synchronize do
|
|
41
|
-
raise 'Cannot configure pool while running' if @running
|
|
42
|
-
|
|
43
|
-
validated_max = max_threads.to_i.clamp(MIN_THREADS, MAX_THREADS)
|
|
44
|
-
@max_threads = validated_max
|
|
45
|
-
@configured = true
|
|
46
|
-
|
|
47
|
-
log('Info', "[SchemaPollingPool] Configured with max_threads: #{@max_threads}")
|
|
48
|
-
end
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
def register?(client_id, client)
|
|
52
|
-
should_start = false
|
|
53
|
-
|
|
54
|
-
@mutex.synchronize do
|
|
55
|
-
if @clients.key?(client_id)
|
|
56
|
-
log('Warn', "[SchemaPollingPool] Client #{client_id} already registered, skipping")
|
|
57
|
-
return false
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
@clients[client_id] = {
|
|
61
|
-
client: client,
|
|
62
|
-
last_poll_at: nil,
|
|
63
|
-
next_poll_at: calculate_initial_poll_time
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
log('Info', "[SchemaPollingPool] Registered client: #{client_id} (#{@clients.size} total clients)")
|
|
67
|
-
|
|
68
|
-
should_start = !@running
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
start_pool if should_start
|
|
72
|
-
|
|
73
|
-
true
|
|
74
|
-
end
|
|
75
|
-
|
|
76
|
-
def unregister?(client_id)
|
|
77
|
-
should_stop = false
|
|
78
|
-
|
|
79
|
-
@mutex.synchronize do
|
|
80
|
-
unless @clients.key?(client_id)
|
|
81
|
-
log('Debug', "[SchemaPollingPool] Client #{client_id} not found for unregister")
|
|
82
|
-
return false
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
@clients.delete(client_id)
|
|
86
|
-
log('Info', "[SchemaPollingPool] Unregistered client: #{client_id} (#{@clients.size} remaining)")
|
|
87
|
-
|
|
88
|
-
should_stop = @clients.empty? && @running
|
|
89
|
-
end
|
|
90
|
-
|
|
91
|
-
stop_pool if should_stop
|
|
92
|
-
|
|
93
|
-
true
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
def client_count
|
|
97
|
-
@mutex.synchronize { @clients.size }
|
|
98
|
-
end
|
|
99
|
-
|
|
100
|
-
def running?
|
|
101
|
-
@mutex.synchronize { @running }
|
|
102
|
-
end
|
|
103
|
-
|
|
104
|
-
def shutdown!
|
|
105
|
-
@mutex.synchronize do
|
|
106
|
-
return unless @running
|
|
107
|
-
|
|
108
|
-
@shutdown_requested = true
|
|
109
|
-
end
|
|
110
|
-
|
|
111
|
-
stop_pool
|
|
112
|
-
|
|
113
|
-
@mutex.synchronize do
|
|
114
|
-
@clients.clear
|
|
115
|
-
@shutdown_requested = false
|
|
116
|
-
end
|
|
117
|
-
end
|
|
118
|
-
|
|
119
|
-
def reset!
|
|
120
|
-
shutdown!
|
|
121
|
-
@mutex.synchronize do
|
|
122
|
-
@max_threads = DEFAULT_MAX_THREADS
|
|
123
|
-
@configured = false
|
|
124
|
-
end
|
|
125
|
-
end
|
|
126
|
-
|
|
127
|
-
private
|
|
128
|
-
|
|
129
|
-
def start_pool
|
|
130
|
-
@mutex.synchronize do
|
|
131
|
-
return if @running
|
|
132
|
-
|
|
133
|
-
@running = true
|
|
134
|
-
@shutdown_requested = false
|
|
135
|
-
|
|
136
|
-
thread_count = @clients.size.clamp(MIN_THREADS, @max_threads)
|
|
137
|
-
|
|
138
|
-
log('Info',
|
|
139
|
-
"[SchemaPollingPool] Starting pool with #{thread_count} worker threads for #{@clients.size} clients")
|
|
140
|
-
|
|
141
|
-
thread_count.times do |i|
|
|
142
|
-
@workers << Thread.new { worker_loop(i) }
|
|
143
|
-
end
|
|
144
|
-
|
|
145
|
-
@scheduler_thread = Thread.new { scheduler_loop }
|
|
146
|
-
end
|
|
147
|
-
end
|
|
148
|
-
|
|
149
|
-
def stop_pool
|
|
150
|
-
workers_to_join = nil
|
|
151
|
-
scheduler_to_join = nil
|
|
152
|
-
|
|
153
|
-
@mutex.synchronize do
|
|
154
|
-
return unless @running
|
|
155
|
-
|
|
156
|
-
log('Info', '[SchemaPollingPool] Stopping pool...')
|
|
157
|
-
|
|
158
|
-
@running = false
|
|
159
|
-
|
|
160
|
-
@workers.size.times { @work_queue << nil }
|
|
161
|
-
|
|
162
|
-
workers_to_join = @workers.dup
|
|
163
|
-
scheduler_to_join = @scheduler_thread
|
|
164
|
-
|
|
165
|
-
@workers.clear
|
|
166
|
-
@scheduler_thread = nil
|
|
167
|
-
end
|
|
168
|
-
|
|
169
|
-
workers_to_join&.each { |w| w.join(2) }
|
|
170
|
-
scheduler_to_join&.join(2)
|
|
171
|
-
|
|
172
|
-
@work_queue.clear
|
|
173
|
-
|
|
174
|
-
log('Info', '[SchemaPollingPool] Pool stopped')
|
|
175
|
-
end
|
|
176
|
-
|
|
177
|
-
def worker_loop(worker_id)
|
|
178
|
-
log('Debug', "[SchemaPollingPool] Worker #{worker_id} started")
|
|
179
|
-
|
|
180
|
-
loop do
|
|
181
|
-
task = fetch_next_task
|
|
182
|
-
break if task.nil?
|
|
183
|
-
|
|
184
|
-
process_task(task, worker_id)
|
|
185
|
-
Thread.pass
|
|
186
|
-
end
|
|
187
|
-
|
|
188
|
-
log('Debug', "[SchemaPollingPool] Worker #{worker_id} stopped")
|
|
189
|
-
end
|
|
190
|
-
|
|
191
|
-
def fetch_next_task
|
|
192
|
-
@work_queue.pop(true)
|
|
193
|
-
rescue ThreadError
|
|
194
|
-
Thread.pass
|
|
195
|
-
sleep(0.1)
|
|
196
|
-
retry if @running
|
|
197
|
-
nil
|
|
198
|
-
end
|
|
199
|
-
|
|
200
|
-
def process_task(task, worker_id)
|
|
201
|
-
client_id = task[:client_id]
|
|
202
|
-
execute_poll(client_id)
|
|
203
|
-
rescue StandardError => e
|
|
204
|
-
log('Error',
|
|
205
|
-
"[SchemaPollingPool] Worker #{worker_id} error polling #{client_id}: #{e.class} - #{e.message}")
|
|
206
|
-
end
|
|
207
|
-
|
|
208
|
-
def scheduler_loop
|
|
209
|
-
log('Debug', '[SchemaPollingPool] Scheduler started')
|
|
210
|
-
|
|
211
|
-
while @running
|
|
212
|
-
sleep_with_check(SCHEDULER_INTERVAL)
|
|
213
|
-
|
|
214
|
-
next unless @running
|
|
215
|
-
|
|
216
|
-
schedule_due_polls
|
|
217
|
-
end
|
|
218
|
-
|
|
219
|
-
log('Debug', '[SchemaPollingPool] Scheduler stopped')
|
|
220
|
-
end
|
|
221
|
-
|
|
222
|
-
def sleep_with_check(duration)
|
|
223
|
-
remaining = duration
|
|
224
|
-
while remaining.positive? && @running
|
|
225
|
-
sleep_time = [remaining, 1.0].min
|
|
226
|
-
sleep(sleep_time)
|
|
227
|
-
remaining -= sleep_time
|
|
228
|
-
Thread.pass
|
|
229
|
-
end
|
|
230
|
-
end
|
|
231
|
-
|
|
232
|
-
def schedule_due_polls
|
|
233
|
-
now = Time.now
|
|
234
|
-
polls_to_schedule = []
|
|
235
|
-
|
|
236
|
-
@mutex.synchronize do
|
|
237
|
-
@clients.each do |client_id, state|
|
|
238
|
-
next if state[:next_poll_at].nil?
|
|
239
|
-
next if now < state[:next_poll_at]
|
|
240
|
-
|
|
241
|
-
polls_to_schedule << client_id
|
|
242
|
-
|
|
243
|
-
interval = state[:client].instance_variable_get(:@polling_interval) || 600
|
|
244
|
-
state[:next_poll_at] = now + interval
|
|
245
|
-
end
|
|
246
|
-
end
|
|
247
|
-
|
|
248
|
-
polls_to_schedule.each do |client_id|
|
|
249
|
-
@work_queue << { client_id: client_id, scheduled_at: now }
|
|
250
|
-
end
|
|
251
|
-
end
|
|
252
|
-
|
|
253
|
-
def execute_poll(client_id)
|
|
254
|
-
client = nil
|
|
255
|
-
@mutex.synchronize do
|
|
256
|
-
state = @clients[client_id]
|
|
257
|
-
client = state[:client] if state
|
|
258
|
-
end
|
|
259
|
-
|
|
260
|
-
return unless client
|
|
261
|
-
return if client.closed
|
|
262
|
-
|
|
263
|
-
log('Debug', "[SchemaPollingPool] Polling client: #{client_id}")
|
|
264
|
-
|
|
265
|
-
client.check_schema
|
|
266
|
-
|
|
267
|
-
@mutex.synchronize do
|
|
268
|
-
@clients[client_id][:last_poll_at] = Time.now if @clients[client_id]
|
|
269
|
-
end
|
|
270
|
-
end
|
|
271
|
-
|
|
272
|
-
def calculate_initial_poll_time
|
|
273
|
-
# Stagger initial polls to avoid thundering herd
|
|
274
|
-
Time.now + rand(0.0..INITIAL_STAGGER_WINDOW.to_f)
|
|
275
|
-
end
|
|
276
|
-
|
|
277
|
-
def log(level, message)
|
|
278
|
-
return unless defined?(ForestAdminAgent::Facades::Container)
|
|
279
|
-
|
|
280
|
-
ForestAdminAgent::Facades::Container.logger&.log(level, message)
|
|
281
|
-
rescue StandardError
|
|
282
|
-
# Ignore logging errors to prevent cascading failures
|
|
283
|
-
end
|
|
284
|
-
end
|
|
285
|
-
end
|
|
286
|
-
end
|