ez_logs_agent 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +80 -0
- data/LICENSE.txt +21 -0
- data/README.md +1023 -0
- data/lib/ez_logs_agent/actor.rb +57 -0
- data/lib/ez_logs_agent/actor_validator.rb +58 -0
- data/lib/ez_logs_agent/buffer.rb +83 -0
- data/lib/ez_logs_agent/capturers/active_job_capturer.rb +300 -0
- data/lib/ez_logs_agent/capturers/database_capturer.rb +467 -0
- data/lib/ez_logs_agent/capturers/job_capturer.rb +261 -0
- data/lib/ez_logs_agent/configuration.rb +184 -0
- data/lib/ez_logs_agent/configuration_validator.rb +139 -0
- data/lib/ez_logs_agent/correlation.rb +40 -0
- data/lib/ez_logs_agent/event_builder.rb +281 -0
- data/lib/ez_logs_agent/flush_scheduler.rb +99 -0
- data/lib/ez_logs_agent/logger.rb +62 -0
- data/lib/ez_logs_agent/middleware/http_request.rb +992 -0
- data/lib/ez_logs_agent/railtie.rb +353 -0
- data/lib/ez_logs_agent/resource_extractor.rb +172 -0
- data/lib/ez_logs_agent/retry_sender.rb +120 -0
- data/lib/ez_logs_agent/sanitizer.rb +150 -0
- data/lib/ez_logs_agent/transport.rb +91 -0
- data/lib/ez_logs_agent/user_agent_detector.rb +51 -0
- data/lib/ez_logs_agent/version.rb +5 -0
- data/lib/ez_logs_agent.rb +44 -0
- data/lib/generators/ez_logs_agent/install/install_generator.rb +94 -0
- data/lib/generators/ez_logs_agent/install/templates/ez_logs_agent.rb.tt +128 -0
- data/lib/tasks/ez_logs_agent.rake +110 -0
- metadata +172 -0
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module EzLogsAgent
|
|
4
|
+
module Capturers
|
|
5
|
+
class JobCapturer
|
|
6
|
+
# Sidekiq client middleware that propagates correlation_id
|
|
7
|
+
# from the current request context into enqueued job payloads.
|
|
8
|
+
#
|
|
9
|
+
# This middleware runs at enqueue-time (when a job is scheduled),
|
|
10
|
+
# NOT at execution-time.
|
|
11
|
+
#
|
|
12
|
+
# == Behavior
|
|
13
|
+
#
|
|
14
|
+
# - Reads EzLogsAgent::Correlation.current
|
|
15
|
+
# - If present, injects it into job["correlation_id"]
|
|
16
|
+
# - Never overwrites existing job["correlation_id"]
|
|
17
|
+
# - Never raises exceptions
|
|
18
|
+
# - Always yields to next middleware
|
|
19
|
+
#
|
|
20
|
+
# == Registration
|
|
21
|
+
#
|
|
22
|
+
# Note: This middleware is automatically registered by Railtie.
|
|
23
|
+
# Users do NOT need to manually configure it.
|
|
24
|
+
#
|
|
25
|
+
# For manual setup (if not using Rails), add to your Sidekiq initializer:
|
|
26
|
+
#
|
|
27
|
+
# Sidekiq.configure_client do |config|
|
|
28
|
+
# config.client_middleware do |chain|
|
|
29
|
+
# chain.add EzLogsAgent::Capturers::JobCapturer::ClientMiddleware
|
|
30
|
+
# end
|
|
31
|
+
# end
|
|
32
|
+
#
|
|
33
|
+
class ClientMiddleware
|
|
34
|
+
# Sidekiq client middleware hook.
|
|
35
|
+
#
|
|
36
|
+
# @param worker_class [Class] The worker class being enqueued
|
|
37
|
+
# @param job [Hash] The Sidekiq job payload (mutable)
|
|
38
|
+
# @param queue [String] The queue name
|
|
39
|
+
# @param redis_pool [ConnectionPool] Sidekiq's Redis connection pool
|
|
40
|
+
# @yield Passes control to the next middleware in the chain
|
|
41
|
+
# @return [void]
|
|
42
|
+
def call(worker_class, job, queue, redis_pool)
|
|
43
|
+
# Defensive: ensure job is a Hash
|
|
44
|
+
return yield unless job.is_a?(Hash)
|
|
45
|
+
|
|
46
|
+
begin
|
|
47
|
+
# Read current correlation_id from context
|
|
48
|
+
correlation_id = EzLogsAgent::Correlation.current
|
|
49
|
+
|
|
50
|
+
# Only inject if:
|
|
51
|
+
# 1. correlation_id is present and not empty
|
|
52
|
+
# 2. job doesn't already have one
|
|
53
|
+
if correlation_id && correlation_id.respond_to?(:empty?) && !correlation_id.empty? && !job.key?("correlation_id")
|
|
54
|
+
job["correlation_id"] = correlation_id
|
|
55
|
+
end
|
|
56
|
+
rescue StandardError => e
|
|
57
|
+
# Never crash the host application during correlation injection
|
|
58
|
+
# Log error, then continue
|
|
59
|
+
EzLogsAgent::Logger.error("[JobCapturer::ClientMiddleware] Error during correlation injection: #{e.class} - #{e.message}")
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Always yield to next middleware
|
|
63
|
+
yield
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Sidekiq server middleware that captures background job execution
|
|
68
|
+
# as background_job events.
|
|
69
|
+
#
|
|
70
|
+
# This middleware runs at execution-time (when the job runs),
|
|
71
|
+
# NOT at enqueue-time.
|
|
72
|
+
#
|
|
73
|
+
# == Behavior
|
|
74
|
+
#
|
|
75
|
+
# - Restores correlation_id from job["correlation_id"] (or generates new one)
|
|
76
|
+
# - Measures job execution duration
|
|
77
|
+
# - Captures success or failure outcome
|
|
78
|
+
# - Re-raises exceptions after capturing failure
|
|
79
|
+
# - Always clears correlation_id in ensure block
|
|
80
|
+
# - Respects capture_jobs configuration flag
|
|
81
|
+
#
|
|
82
|
+
# == Registration
|
|
83
|
+
#
|
|
84
|
+
# Note: This middleware is automatically registered by Railtie.
|
|
85
|
+
# Users do NOT need to manually configure it.
|
|
86
|
+
#
|
|
87
|
+
# For manual setup (if not using Rails), add to your Sidekiq initializer:
|
|
88
|
+
#
|
|
89
|
+
# Sidekiq.configure_server do |config|
|
|
90
|
+
# config.server_middleware do |chain|
|
|
91
|
+
# chain.add EzLogsAgent::Capturers::JobCapturer::ServerMiddleware
|
|
92
|
+
# end
|
|
93
|
+
# end
|
|
94
|
+
#
|
|
95
|
+
class ServerMiddleware
|
|
96
|
+
# Sidekiq server middleware hook.
|
|
97
|
+
#
|
|
98
|
+
# @param worker [Object] The worker instance executing the job
|
|
99
|
+
# @param job [Hash] The Sidekiq job payload
|
|
100
|
+
# @param queue [String] The queue name
|
|
101
|
+
# @yield Executes the job
|
|
102
|
+
# @return [void]
|
|
103
|
+
def call(worker, job, queue)
|
|
104
|
+
# Skip capture if disabled
|
|
105
|
+
return yield unless EzLogsAgent.configuration.capture_jobs
|
|
106
|
+
|
|
107
|
+
# Skip excluded job classes (health checks, infrastructure jobs).
|
|
108
|
+
# Match against the underlying ActiveJob class too, not just the wrapper.
|
|
109
|
+
if excluded_job_class?(worker, job)
|
|
110
|
+
EzLogsAgent::Logger.debug("[JobCapturer::ServerMiddleware] Skipping capture (excluded job class: #{resolve_job_class(worker, job)})")
|
|
111
|
+
return yield
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Restore correlation_id from job payload (or generate new one)
|
|
115
|
+
correlation_id = job["correlation_id"] || EzLogsAgent::Correlation.generate
|
|
116
|
+
EzLogsAgent::Correlation.current = correlation_id
|
|
117
|
+
|
|
118
|
+
# Measure execution time
|
|
119
|
+
start_time = Time.now
|
|
120
|
+
result = yield
|
|
121
|
+
duration_ms = ((Time.now - start_time) * 1000).to_i
|
|
122
|
+
|
|
123
|
+
# Capture success event
|
|
124
|
+
capture_success(worker, job, queue, correlation_id, duration_ms, start_time)
|
|
125
|
+
|
|
126
|
+
result
|
|
127
|
+
rescue StandardError => error
|
|
128
|
+
# Capture failure event, then re-raise
|
|
129
|
+
capture_failure(worker, job, queue, correlation_id, error, start_time)
|
|
130
|
+
raise
|
|
131
|
+
ensure
|
|
132
|
+
# Always clear correlation_id
|
|
133
|
+
EzLogsAgent::Correlation.clear
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
private
|
|
137
|
+
|
|
138
|
+
# Captures successful job execution.
|
|
139
|
+
#
|
|
140
|
+
# @param worker [Object] The worker instance
|
|
141
|
+
# @param job [Hash] The Sidekiq job payload
|
|
142
|
+
# @param queue [String] The queue name
|
|
143
|
+
# @param correlation_id [String] The correlation ID
|
|
144
|
+
# @param duration_ms [Integer] Job execution duration in milliseconds
|
|
145
|
+
# @param start_time [Time] Job start time
|
|
146
|
+
# @return [void]
|
|
147
|
+
def capture_success(worker, job, queue, correlation_id, duration_ms, start_time)
|
|
148
|
+
event = EzLogsAgent::EventBuilder.build(
|
|
149
|
+
source_type: :background_job,
|
|
150
|
+
source_data: extract_job_data(worker, job, queue),
|
|
151
|
+
outcome: :success,
|
|
152
|
+
correlation_id: correlation_id,
|
|
153
|
+
duration_ms: duration_ms,
|
|
154
|
+
timestamp: start_time
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
EzLogsAgent::Buffer.push(event)
|
|
158
|
+
rescue StandardError => e
|
|
159
|
+
# Defensive: never crash the host application during event capture
|
|
160
|
+
EzLogsAgent::Logger.error("[JobCapturer::ServerMiddleware] Failed to capture success event: #{e.class} - #{e.message}")
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Captures failed job execution.
|
|
164
|
+
#
|
|
165
|
+
# @param worker [Object] The worker instance
|
|
166
|
+
# @param job [Hash] The Sidekiq job payload
|
|
167
|
+
# @param queue [String] The queue name
|
|
168
|
+
# @param correlation_id [String] The correlation ID
|
|
169
|
+
# @param error [StandardError] The error that caused the failure
|
|
170
|
+
# @param start_time [Time] Job start time
|
|
171
|
+
# @return [void]
|
|
172
|
+
def capture_failure(worker, job, queue, correlation_id, error, start_time)
|
|
173
|
+
event = EzLogsAgent::EventBuilder.build(
|
|
174
|
+
source_type: :background_job,
|
|
175
|
+
source_data: extract_job_data(worker, job, queue),
|
|
176
|
+
outcome: :failure,
|
|
177
|
+
correlation_id: correlation_id,
|
|
178
|
+
error_message: "#{error.class}: #{error.message}",
|
|
179
|
+
timestamp: start_time
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
EzLogsAgent::Buffer.push(event)
|
|
183
|
+
rescue StandardError => e
|
|
184
|
+
# Defensive: never crash the host application during event capture
|
|
185
|
+
EzLogsAgent::Logger.error("[JobCapturer::ServerMiddleware] Failed to capture failure event: #{e.class} - #{e.message}")
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# Extracts relevant job data for event source_data.
|
|
189
|
+
#
|
|
190
|
+
# When the worker is the ActiveJob+Sidekiq wrapper, we surface the
|
|
191
|
+
# underlying ActiveJob class name (e.g., "ChargeCardJob") rather
|
|
192
|
+
# than the wrapper class. This mirrors how Sidekiq itself resolves
|
|
193
|
+
# the display class — see Sidekiq::JobLogger and Sidekiq::JobUtil:
|
|
194
|
+
#
|
|
195
|
+
# job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"]
|
|
196
|
+
#
|
|
197
|
+
# @param worker [Object] The worker instance
|
|
198
|
+
# @param job [Hash] The Sidekiq job payload
|
|
199
|
+
# @param queue [String] The queue name
|
|
200
|
+
# @return [Hash] Job metadata for source_data
|
|
201
|
+
def extract_job_data(worker, job, queue)
|
|
202
|
+
{
|
|
203
|
+
job_class: resolve_job_class(worker, job),
|
|
204
|
+
job_id: job["jid"],
|
|
205
|
+
queue: queue,
|
|
206
|
+
retry_count: job["retry_count"],
|
|
207
|
+
arguments: extract_arguments(job)
|
|
208
|
+
}.compact
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# Sanitize the job's arguments. Sidekiq stores them in
|
|
212
|
+
# `job["args"]`. When the job is wrapped by ActiveJob, the
|
|
213
|
+
# outer args is a single-element array containing an
|
|
214
|
+
# ActiveJob payload whose `arguments` key holds the actual
|
|
215
|
+
# user-passed args — unwrap that so the dashboard shows the
|
|
216
|
+
# arguments the user wrote, not the ActiveJob plumbing.
|
|
217
|
+
def extract_arguments(job)
|
|
218
|
+
raw = job["args"]
|
|
219
|
+
return nil if raw.nil? || (raw.respond_to?(:empty?) && raw.empty?)
|
|
220
|
+
|
|
221
|
+
# ActiveJob+Sidekiq: args == [{"job_class"=>..., "arguments"=>[...], ...}]
|
|
222
|
+
if raw.is_a?(Array) && raw.size == 1 && raw.first.is_a?(Hash) && raw.first.key?("arguments")
|
|
223
|
+
raw = raw.first["arguments"]
|
|
224
|
+
return nil if raw.nil? || (raw.respond_to?(:empty?) && raw.empty?)
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
EzLogsAgent::Sanitizer.sanitize_args(raw)
|
|
228
|
+
rescue StandardError => e
|
|
229
|
+
EzLogsAgent::Logger.debug("[JobCapturer::ServerMiddleware] argument extraction failed: #{e.class}: #{e.message}")
|
|
230
|
+
nil
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
# Resolves the human-meaningful job class name, unwrapping the
|
|
234
|
+
# ActiveJob+Sidekiq adapter when present.
|
|
235
|
+
#
|
|
236
|
+
# @param worker [Object] The worker instance Sidekiq is running
|
|
237
|
+
# @param job [Hash] The Sidekiq job payload
|
|
238
|
+
# @return [String] The class name to record on the event
|
|
239
|
+
def resolve_job_class(worker, job)
|
|
240
|
+
job["display_class"] || job["wrapped"] || worker.class.name
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# Checks if worker class is in the excluded list.
|
|
244
|
+
#
|
|
245
|
+
# Exclusion is matched against the same name that lands in
|
|
246
|
+
# source_data, so users can list either the Sidekiq worker class
|
|
247
|
+
# or the underlying ActiveJob class — whichever they actually own.
|
|
248
|
+
#
|
|
249
|
+
# @param worker [Object] The worker instance
|
|
250
|
+
# @param job [Hash] The Sidekiq job payload (optional; defaults to {})
|
|
251
|
+
# @return [Boolean] true if excluded, false otherwise
|
|
252
|
+
def excluded_job_class?(worker, job = {})
|
|
253
|
+
excluded = EzLogsAgent.configuration.all_excluded_job_classes
|
|
254
|
+
[worker.class.name, job["wrapped"], job["display_class"]].compact.any? { |name| excluded.include?(name) }
|
|
255
|
+
rescue StandardError
|
|
256
|
+
false
|
|
257
|
+
end
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
end
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module EzLogsAgent
|
|
4
|
+
# Configuration for the EzLogsAgent.
|
|
5
|
+
#
|
|
6
|
+
# == Noise Filtering Overview
|
|
7
|
+
#
|
|
8
|
+
# EzLogsAgent captures events but filters out noise at the agent level.
|
|
9
|
+
# Each capturer has its own exclusion mechanism:
|
|
10
|
+
#
|
|
11
|
+
# === HTTP Requests (middleware/http_request.rb)
|
|
12
|
+
# - excluded_paths: URL paths to ignore (supports * wildcard for prefix match)
|
|
13
|
+
# - DEFAULT_EXCLUDED_EXTENSIONS: Static file extensions (.js, .css, .png, etc.)
|
|
14
|
+
# - excluded_graphql_operations: GraphQL operations to skip (introspection, etc.)
|
|
15
|
+
#
|
|
16
|
+
# === Database Callbacks (capturers/database_capturer.rb)
|
|
17
|
+
# - excluded_tables: Table names to ignore (Rails internals, job queues, etc.)
|
|
18
|
+
# - IGNORED_ATTRIBUTES: Technical fields (created_at, lock_version, etc.)
|
|
19
|
+
# - SENSITIVE_PATTERNS: Attributes containing passwords, tokens, secrets
|
|
20
|
+
#
|
|
21
|
+
# === Background Jobs (capturers/job_capturer.rb)
|
|
22
|
+
# - excluded_job_classes: Job class names to ignore (health checks, etc.)
|
|
23
|
+
#
|
|
24
|
+
# == Server-Side vs Agent-Side Filtering
|
|
25
|
+
#
|
|
26
|
+
# The filtering philosophy:
|
|
27
|
+
# - Agent filters NOISE (introspection, assets, health checks, Rails internals)
|
|
28
|
+
# - Server classifies SIGNIFICANCE (reads vs writes) for UI filtering
|
|
29
|
+
#
|
|
30
|
+
# This means GraphQL queries and GET requests ARE captured by the agent,
|
|
31
|
+
# but the server classifies them as "background" so users can toggle visibility.
|
|
32
|
+
#
|
|
33
|
+
class Configuration
|
|
34
|
+
attr_accessor :server_url
|
|
35
|
+
attr_accessor :project_token
|
|
36
|
+
attr_accessor :capture_http
|
|
37
|
+
attr_accessor :capture_jobs
|
|
38
|
+
attr_accessor :capture_database
|
|
39
|
+
attr_accessor :excluded_paths
|
|
40
|
+
attr_accessor :excluded_tables
|
|
41
|
+
attr_accessor :excluded_job_classes
|
|
42
|
+
attr_accessor :excluded_graphql_operations
|
|
43
|
+
attr_accessor :excluded_graphql_variable_keys
|
|
44
|
+
attr_accessor :buffer_size
|
|
45
|
+
attr_accessor :retry_attempts
|
|
46
|
+
attr_accessor :send_interval
|
|
47
|
+
attr_accessor :log_level
|
|
48
|
+
|
|
49
|
+
# Actor extraction hook for HTTP requests (optional)
|
|
50
|
+
# Must be a callable (lambda/proc) that accepts (request, controller)
|
|
51
|
+
# and returns { kind:, id:, label:, metadata: } or nil
|
|
52
|
+
attr_accessor :actor_from_request
|
|
53
|
+
|
|
54
|
+
# Display name field mapping for database records (optional)
|
|
55
|
+
# Maps model class names to attribute names used for human-readable display
|
|
56
|
+
#
|
|
57
|
+
# Example:
|
|
58
|
+
# config.display_name_for = {
|
|
59
|
+
# "User" => :email,
|
|
60
|
+
# "Product" => :name,
|
|
61
|
+
# "Order" => :number
|
|
62
|
+
# }
|
|
63
|
+
#
|
|
64
|
+
# IMPORTANT: Only use direct attributes, not associations.
|
|
65
|
+
# Associations will trigger database queries and should be avoided.
|
|
66
|
+
#
|
|
67
|
+
# If not configured for a model, falls back to: name → title → number → "##{id}"
|
|
68
|
+
attr_accessor :display_name_for
|
|
69
|
+
|
|
70
|
+
# Default paths excluded from HTTP capture - common Rails noise
|
|
71
|
+
DEFAULT_EXCLUDED_PATHS = [
|
|
72
|
+
"/rails/active_storage*", # File uploads/downloads
|
|
73
|
+
"/assets*", # Asset pipeline
|
|
74
|
+
"/packs*", # Webpacker assets
|
|
75
|
+
"/vite*", # Vite assets
|
|
76
|
+
"/health*", # Health checks
|
|
77
|
+
"/up", # Rails 7.1+ health check
|
|
78
|
+
"/alive", # Kubernetes liveness probe
|
|
79
|
+
"/ready", # Kubernetes readiness probe
|
|
80
|
+
"/metrics", # Prometheus metrics endpoint
|
|
81
|
+
"/favicon.ico", # Browser favicon
|
|
82
|
+
"/*.hot-update.*", # Hot module replacement
|
|
83
|
+
"/.well-known*", # Well-known URIs (security.txt, etc.)
|
|
84
|
+
"/robots.txt", # Search engine crawler config
|
|
85
|
+
"/sitemap.xml", # Sitemap for crawlers
|
|
86
|
+
"/cable*", # ActionCable WebSocket connections
|
|
87
|
+
"/sidekiq", # Sidekiq Web UI dashboard root (the conventional mount)
|
|
88
|
+
"/sidekiq/*", # Sidekiq Web UI sub-paths (auto-poll noise)
|
|
89
|
+
# Authentication pages - not meaningful business actions
|
|
90
|
+
# Use */path* patterns to match auth routes anywhere (e.g., /admin/logout)
|
|
91
|
+
"*/sign_in*", # Devise and common sign in (matches /users/sign_in, /admin/sign_in)
|
|
92
|
+
"*/sign_out*", # Devise and common sign out
|
|
93
|
+
"*/login*", # Common auth pattern (matches /login, /admin/login)
|
|
94
|
+
"*/logout*", # Common auth pattern (matches /logout, /admin/logout)
|
|
95
|
+
"/users/password*", # Devise password reset/edit
|
|
96
|
+
"/session*" # Common auth pattern
|
|
97
|
+
].freeze
|
|
98
|
+
|
|
99
|
+
# Default file extensions excluded from HTTP capture - static assets
|
|
100
|
+
# These are matched against the path suffix regardless of directory
|
|
101
|
+
DEFAULT_EXCLUDED_EXTENSIONS = %w[
|
|
102
|
+
.js .css .map
|
|
103
|
+
.png .jpg .jpeg .gif .svg .ico .webp
|
|
104
|
+
.woff .woff2 .ttf .eot .otf
|
|
105
|
+
].freeze
|
|
106
|
+
|
|
107
|
+
# Default tables excluded from database capture - Rails internal tables
|
|
108
|
+
DEFAULT_EXCLUDED_TABLES = [
|
|
109
|
+
"schema_migrations",
|
|
110
|
+
"ar_internal_metadata",
|
|
111
|
+
"sessions", # ActiveRecord session store (plural)
|
|
112
|
+
"session", # ActiveRecord session store (singular)
|
|
113
|
+
"active_storage_blobs", # ActiveStorage internals
|
|
114
|
+
"active_storage_attachments",
|
|
115
|
+
"active_storage_variant_records",
|
|
116
|
+
"solid_queue_jobs", # SolidQueue internals
|
|
117
|
+
"solid_queue_scheduled_executions",
|
|
118
|
+
"solid_queue_ready_executions",
|
|
119
|
+
"solid_queue_claimed_executions",
|
|
120
|
+
"solid_queue_blocked_executions",
|
|
121
|
+
"solid_queue_failed_executions",
|
|
122
|
+
"solid_queue_pauses",
|
|
123
|
+
"solid_queue_processes",
|
|
124
|
+
"solid_queue_semaphores",
|
|
125
|
+
"solid_queue_recurring_tasks",
|
|
126
|
+
"solid_queue_recurring_executions",
|
|
127
|
+
"solid_cache_entries", # SolidCache internals
|
|
128
|
+
"solid_cable_messages" # SolidCable internals
|
|
129
|
+
].freeze
|
|
130
|
+
|
|
131
|
+
# Default job classes excluded from background job capture - infrastructure/health check jobs
|
|
132
|
+
DEFAULT_EXCLUDED_JOB_CLASSES = [
|
|
133
|
+
"SidekiqAlive::Worker", # Sidekiq health check
|
|
134
|
+
"SolidQueue::CleanupJob", # SolidQueue maintenance
|
|
135
|
+
"SolidQueue::RecurringJob" # SolidQueue scheduler internals
|
|
136
|
+
].freeze
|
|
137
|
+
|
|
138
|
+
# Default GraphQL operations excluded from capture - introspection and IDE queries
|
|
139
|
+
# Supports exact match and prefix match (patterns ending with *)
|
|
140
|
+
DEFAULT_EXCLUDED_GRAPHQL_OPERATIONS = [
|
|
141
|
+
"IntrospectionQuery", # Standard IDE introspection query
|
|
142
|
+
"__*" # All introspection fields (__schema, __type, etc.)
|
|
143
|
+
].freeze
|
|
144
|
+
|
|
145
|
+
def initialize
|
|
146
|
+
@server_url = nil
|
|
147
|
+
@project_token = nil
|
|
148
|
+
@capture_http = true
|
|
149
|
+
@capture_jobs = true
|
|
150
|
+
@capture_database = true
|
|
151
|
+
@excluded_paths = [] # User-defined; combined with DEFAULT_EXCLUDED_PATHS
|
|
152
|
+
@excluded_tables = [] # User-defined; combined with DEFAULT_EXCLUDED_TABLES
|
|
153
|
+
@excluded_job_classes = [] # User-defined; combined with DEFAULT_EXCLUDED_JOB_CLASSES
|
|
154
|
+
@excluded_graphql_operations = [] # User-defined; combined with DEFAULT_EXCLUDED_GRAPHQL_OPERATIONS
|
|
155
|
+
@excluded_graphql_variable_keys = [] # User-defined; additional sensitive variable key patterns to filter
|
|
156
|
+
@buffer_size = 10_000 # Increased for high-volume workloads (job-heavy apps)
|
|
157
|
+
@retry_attempts = 3
|
|
158
|
+
@send_interval = 3 # More frequent sends for better throughput
|
|
159
|
+
@log_level = :warn
|
|
160
|
+
@actor_from_request = nil # Not configured by default
|
|
161
|
+
@display_name_for = {} # Not configured by default
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# Returns all excluded paths (defaults + user-configured)
|
|
165
|
+
def all_excluded_paths
|
|
166
|
+
DEFAULT_EXCLUDED_PATHS + (@excluded_paths || [])
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Returns all excluded tables (defaults + user-configured)
|
|
170
|
+
def all_excluded_tables
|
|
171
|
+
DEFAULT_EXCLUDED_TABLES + (@excluded_tables || [])
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# Returns all excluded job classes (defaults + user-configured)
|
|
175
|
+
def all_excluded_job_classes
|
|
176
|
+
DEFAULT_EXCLUDED_JOB_CLASSES + (@excluded_job_classes || [])
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# Returns all excluded GraphQL operations (defaults + user-configured)
|
|
180
|
+
def all_excluded_graphql_operations
|
|
181
|
+
DEFAULT_EXCLUDED_GRAPHQL_OPERATIONS + (@excluded_graphql_operations || [])
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
end
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module EzLogsAgent
|
|
4
|
+
# Validates EzLogsAgent configuration and provides actionable error messages.
|
|
5
|
+
#
|
|
6
|
+
# Used at boot-time by Railtie and by the test_connection rake task
|
|
7
|
+
# to ensure configuration is valid before attempting to capture or send events.
|
|
8
|
+
#
|
|
9
|
+
# Validation philosophy:
|
|
10
|
+
# - Only validate what will cause hard failures
|
|
11
|
+
# - Provide clear, actionable error messages
|
|
12
|
+
# - Warnings for optional but recommended config
|
|
13
|
+
# - Never crash the host application
|
|
14
|
+
#
|
|
15
|
+
class ConfigurationValidator
|
|
16
|
+
# Result of configuration validation
|
|
17
|
+
#
|
|
18
|
+
# @attr_reader [Array<String>] errors Critical configuration errors
|
|
19
|
+
# @attr_reader [Array<String>] warnings Non-critical configuration warnings
|
|
20
|
+
ValidationResult = Struct.new(:errors, :warnings) do
|
|
21
|
+
def valid?
|
|
22
|
+
errors.empty?
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def has_warnings?
|
|
26
|
+
warnings.any?
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Validates the given configuration
|
|
31
|
+
#
|
|
32
|
+
# @param config [EzLogsAgent::Configuration] Configuration to validate
|
|
33
|
+
# @return [ValidationResult] Validation result with errors and warnings
|
|
34
|
+
def self.validate(config)
|
|
35
|
+
errors = []
|
|
36
|
+
warnings = []
|
|
37
|
+
|
|
38
|
+
# Required: server_url must be set
|
|
39
|
+
if config.server_url.nil? || config.server_url.to_s.strip.empty?
|
|
40
|
+
errors << "server_url is required. Set it in config/initializers/ez_logs_agent.rb"
|
|
41
|
+
elsif !valid_url?(config.server_url)
|
|
42
|
+
errors << "server_url must be a valid URL (got: #{config.server_url})"
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Optional but recommended: project_token
|
|
46
|
+
if config.project_token.nil? || config.project_token.to_s.strip.empty?
|
|
47
|
+
warnings << "project_token is not set. Authentication may fail if the server requires it."
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Validate numeric fields
|
|
51
|
+
if config.buffer_size && (!config.buffer_size.is_a?(Integer) || config.buffer_size <= 0)
|
|
52
|
+
errors << "buffer_size must be a positive integer (got: #{config.buffer_size})"
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
if config.retry_attempts && (!config.retry_attempts.is_a?(Integer) || config.retry_attempts < 0)
|
|
56
|
+
errors << "retry_attempts must be a non-negative integer (got: #{config.retry_attempts})"
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
if config.send_interval && (!config.send_interval.is_a?(Numeric) || config.send_interval <= 0)
|
|
60
|
+
errors << "send_interval must be a positive number (got: #{config.send_interval})"
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Validate log_level
|
|
64
|
+
if config.log_level && !valid_log_level?(config.log_level)
|
|
65
|
+
errors << "log_level must be one of: :debug, :info, :warn, :error (got: #{config.log_level})"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Validate boolean fields
|
|
69
|
+
unless [true, false].include?(config.capture_http)
|
|
70
|
+
errors << "capture_http must be true or false (got: #{config.capture_http})"
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
unless [true, false].include?(config.capture_jobs)
|
|
74
|
+
errors << "capture_jobs must be true or false (got: #{config.capture_jobs})"
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
unless [true, false].include?(config.capture_database)
|
|
78
|
+
errors << "capture_database must be true or false (got: #{config.capture_database})"
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Validate array fields
|
|
82
|
+
if config.excluded_paths && !config.excluded_paths.is_a?(Array)
|
|
83
|
+
errors << "excluded_paths must be an array (got: #{config.excluded_paths.class})"
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
if config.excluded_tables && !config.excluded_tables.is_a?(Array)
|
|
87
|
+
errors << "excluded_tables must be an array (got: #{config.excluded_tables.class})"
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
if config.excluded_job_classes && !config.excluded_job_classes.is_a?(Array)
|
|
91
|
+
errors << "excluded_job_classes must be an array (got: #{config.excluded_job_classes.class})"
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
if config.excluded_graphql_operations && !config.excluded_graphql_operations.is_a?(Array)
|
|
95
|
+
errors << "excluded_graphql_operations must be an array (got: #{config.excluded_graphql_operations.class})"
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Validate actor_from_request is callable if present
|
|
99
|
+
if config.actor_from_request && !config.actor_from_request.respond_to?(:call)
|
|
100
|
+
errors << "actor_from_request must be a callable (lambda/proc) or nil"
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Validate display_name_for is a hash if present
|
|
104
|
+
if config.display_name_for && !config.display_name_for.is_a?(Hash)
|
|
105
|
+
errors << "display_name_for must be a hash (got: #{config.display_name_for.class})"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Warning if all capture flags are disabled
|
|
109
|
+
if !config.capture_http && !config.capture_jobs && !config.capture_database
|
|
110
|
+
warnings << "All capture flags are disabled. No events will be captured."
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
ValidationResult.new(errors, warnings)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Checks if the given string is a valid URL
|
|
117
|
+
#
|
|
118
|
+
# @param url [String] URL to validate
|
|
119
|
+
# @return [Boolean] true if valid URL
|
|
120
|
+
def self.valid_url?(url)
|
|
121
|
+
return false unless url.is_a?(String)
|
|
122
|
+
|
|
123
|
+
uri = URI.parse(url)
|
|
124
|
+
uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
|
|
125
|
+
rescue URI::InvalidURIError
|
|
126
|
+
false
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Checks if the given log level is valid
|
|
130
|
+
#
|
|
131
|
+
# @param level [Symbol] Log level to validate
|
|
132
|
+
# @return [Boolean] true if valid log level
|
|
133
|
+
def self.valid_log_level?(level)
|
|
134
|
+
%i[debug info warn error].include?(level)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
private_class_method :valid_url?, :valid_log_level?
|
|
138
|
+
end
|
|
139
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
require "securerandom"
|
|
2
|
+
require "request_store"
|
|
3
|
+
|
|
4
|
+
module EzLogsAgent
|
|
5
|
+
module Correlation
|
|
6
|
+
STORE_KEY = :ez_logs_correlation_id
|
|
7
|
+
|
|
8
|
+
# Generate a new unique correlation ID
|
|
9
|
+
# Format: ezl_<timestamp_ms>_<random_hex_8>
|
|
10
|
+
#
|
|
11
|
+
# @return [String] A new correlation ID
|
|
12
|
+
def self.generate
|
|
13
|
+
timestamp_ms = (Time.now.to_f * 1000).to_i
|
|
14
|
+
random_hex = SecureRandom.hex(4) # 8 chars
|
|
15
|
+
"ezl_#{timestamp_ms}_#{random_hex}"
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Get the current correlation ID (or nil)
|
|
19
|
+
#
|
|
20
|
+
# @return [String, nil] The current correlation ID or nil if not set
|
|
21
|
+
def self.current
|
|
22
|
+
RequestStore.store[STORE_KEY]
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Set the current correlation ID
|
|
26
|
+
#
|
|
27
|
+
# @param value [String, nil] The correlation ID to store
|
|
28
|
+
# @return [String, nil] The stored value
|
|
29
|
+
def self.current=(value)
|
|
30
|
+
RequestStore.store[STORE_KEY] = value
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Clear the current correlation ID
|
|
34
|
+
#
|
|
35
|
+
# @return [void]
|
|
36
|
+
def self.clear
|
|
37
|
+
RequestStore.store.delete(STORE_KEY)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|