flare 0.4.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +60 -1
- data/app/controllers/flare/jobs_controller.rb +1 -1
- data/app/controllers/flare/requests_controller.rb +1 -1
- data/app/controllers/flare/spans_controller.rb +1 -1
- data/app/helpers/flare/application_helper.rb +23 -0
- data/app/views/flare/jobs/index.html.erb +2 -1
- data/app/views/flare/requests/index.html.erb +2 -1
- data/app/views/flare/requests/show.html.erb +16 -7
- data/app/views/flare/spans/index.html.erb +2 -1
- data/lib/flare/cli/doctor_command.rb +4 -11
- data/lib/flare/cli/local_spans.rb +110 -0
- data/lib/flare/cli/nplus1_command.rb +97 -0
- data/lib/flare/cli/project.rb +28 -0
- data/lib/flare/cli/request_command.rb +120 -0
- data/lib/flare/cli/requests_command.rb +45 -0
- data/lib/flare/cli/setup_command.rb +64 -41
- data/lib/flare/cli/status_command.rb +5 -2
- data/lib/flare/cli.rb +12 -0
- data/lib/flare/client_headers.rb +7 -3
- data/lib/flare/configuration.rb +20 -0
- data/lib/flare/export_filter.rb +273 -0
- data/lib/flare/filtering_span_processor.rb +14 -0
- data/lib/flare/http_transport.rb +4 -0
- data/lib/flare/metric_span_processor.rb +51 -35
- data/lib/flare/metric_submitter.rb +9 -1
- data/lib/flare/recording_batch_span_processor.rb +9 -1
- data/lib/flare/redis_statement_processor.rb +29 -0
- data/lib/flare/revision.rb +82 -0
- data/lib/flare/source_location.rb +21 -16
- data/lib/flare/sql_mask.rb +235 -0
- data/lib/flare/sqlite_exporter.rb +12 -8
- data/lib/flare/storage/sqlite.rb +195 -144
- data/lib/flare/storage.rb +1 -40
- data/lib/flare/store_statement.rb +136 -0
- data/lib/flare/trace_blob.rb +9 -5
- data/lib/flare/trace_exporter.rb +18 -3
- data/lib/flare/upload_url_pool.rb +2 -15
- data/lib/flare/utf8.rb +20 -0
- data/lib/flare/version.rb +1 -1
- data/lib/flare.rb +21 -2
- data/public/flare-assets/flare.css +10 -60
- metadata +45 -32
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "local_spans"
|
|
5
|
+
require_relative "output"
|
|
6
|
+
|
|
7
|
+
module Flare
|
|
8
|
+
# Lists the most recent requests from the local development spans database,
|
|
9
|
+
# so you (or a coding agent) can check a change without opening /flare.
|
|
10
|
+
class RequestsCommand
|
|
11
|
+
include CLI::LocalSpans
|
|
12
|
+
include CLI::Output
|
|
13
|
+
|
|
14
|
+
def initialize(argv)
|
|
15
|
+
parse_options(argv, banner: "Usage: flare requests [options]", last_help: "Number of requests to show")
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def run
|
|
19
|
+
requests = read_local_spans(@database_path) { |storage| recent_requests(storage, @limit) }
|
|
20
|
+
|
|
21
|
+
if @json
|
|
22
|
+
puts JSON.pretty_generate(requests)
|
|
23
|
+
elsif requests.empty?
|
|
24
|
+
puts "No requests recorded yet."
|
|
25
|
+
else
|
|
26
|
+
requests.each { |request| puts format_row(request) }
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def format_row(request)
|
|
33
|
+
[
|
|
34
|
+
dim(request[:started_at]),
|
|
35
|
+
request[:method].to_s.ljust(6),
|
|
36
|
+
request[:status].to_s.ljust(3),
|
|
37
|
+
format("%8.1fms", request[:duration_ms]),
|
|
38
|
+
format("%4d queries", request[:queries]),
|
|
39
|
+
request[:path],
|
|
40
|
+
endpoint(request)&.then { |name| dim(name) },
|
|
41
|
+
dim(request[:trace_id]),
|
|
42
|
+
].compact.join(" ")
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -8,14 +8,20 @@ require "net/http"
|
|
|
8
8
|
require "uri"
|
|
9
9
|
require "json"
|
|
10
10
|
require "fileutils"
|
|
11
|
+
require "timeout"
|
|
11
12
|
require_relative "output"
|
|
13
|
+
require_relative "project"
|
|
12
14
|
|
|
13
15
|
module Flare
|
|
14
16
|
class SetupCommand
|
|
15
17
|
include CLI::Output
|
|
18
|
+
include CLI::Project
|
|
16
19
|
|
|
17
20
|
DEFAULT_HOST = "https://flare.am"
|
|
18
21
|
TIMEOUT_SECONDS = 300 # 5 minutes
|
|
22
|
+
CLIENT_READ_TIMEOUT = 1
|
|
23
|
+
|
|
24
|
+
class AuthorizationDenied < StandardError; end
|
|
19
25
|
|
|
20
26
|
INITIALIZER_CONTENT = <<~RUBY
|
|
21
27
|
# frozen_string_literal: true
|
|
@@ -109,9 +115,7 @@ module Flare
|
|
|
109
115
|
# --- Auth ---
|
|
110
116
|
|
|
111
117
|
def authenticate
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
if !@force && File.exist?(env_path) && File.read(env_path).match?(/^FLARE_KEY=.+/)
|
|
118
|
+
if !@force && key_in_dotenv?
|
|
115
119
|
puts "#{checkmark} FLARE_KEY already set in .env, skipping auth."
|
|
116
120
|
puts " Run with --force to re-authenticate."
|
|
117
121
|
return
|
|
@@ -139,7 +143,12 @@ module Flare
|
|
|
139
143
|
puts
|
|
140
144
|
puts "Waiting for authorization (up to 5 minutes)..."
|
|
141
145
|
|
|
142
|
-
|
|
146
|
+
begin
|
|
147
|
+
auth_code = wait_for_callback(server, state, TIMEOUT_SECONDS)
|
|
148
|
+
rescue AuthorizationDenied => e
|
|
149
|
+
puts "Authorization failed: #{e.message}"
|
|
150
|
+
exit 1
|
|
151
|
+
end
|
|
143
152
|
|
|
144
153
|
unless auth_code
|
|
145
154
|
puts "Timed out waiting for authorization."
|
|
@@ -152,50 +161,59 @@ module Flare
|
|
|
152
161
|
server&.close
|
|
153
162
|
end
|
|
154
163
|
|
|
164
|
+
# Waits for flare-web to redirect the browser back with a code. Anything
|
|
165
|
+
# else that reaches the port (a stray request, a wrong or missing state,
|
|
166
|
+
# a connection that never sends a request) gets a response and is
|
|
167
|
+
# skipped, so only a matching callback ends the wait.
|
|
155
168
|
def wait_for_callback(server, expected_state, timeout)
|
|
156
169
|
deadline = Time.now + timeout
|
|
157
170
|
|
|
158
171
|
while Time.now < deadline
|
|
159
|
-
|
|
160
|
-
next unless readable
|
|
172
|
+
next unless IO.select([server], nil, nil, 1)
|
|
161
173
|
|
|
162
|
-
|
|
163
|
-
|
|
174
|
+
code = handle_callback(server.accept, expected_state)
|
|
175
|
+
return code if code
|
|
176
|
+
end
|
|
164
177
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
client.close
|
|
168
|
-
next
|
|
169
|
-
end
|
|
178
|
+
nil
|
|
179
|
+
end
|
|
170
180
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
if error
|
|
180
|
-
client.print http_response(error_page(error))
|
|
181
|
-
client.close
|
|
182
|
-
return nil
|
|
183
|
-
elsif returned_state != expected_state
|
|
184
|
-
client.print http_response(error_page("State mismatch. Please try again."))
|
|
185
|
-
client.close
|
|
186
|
-
return nil
|
|
187
|
-
elsif code.nil? || code.empty?
|
|
188
|
-
client.print http_response(error_page("No authorization code received."))
|
|
189
|
-
client.close
|
|
190
|
-
return nil
|
|
191
|
-
else
|
|
192
|
-
client.print http_response(success_page)
|
|
193
|
-
client.close
|
|
194
|
-
return code
|
|
181
|
+
# Returns the code, nil to keep waiting, or raises AuthorizationDenied
|
|
182
|
+
# when flare-web reports an error for this state.
|
|
183
|
+
def handle_callback(client, expected_state)
|
|
184
|
+
request_line = Timeout.timeout(CLIENT_READ_TIMEOUT) do
|
|
185
|
+
client.gets.tap do
|
|
186
|
+
# Read remaining headers (required by HTTP spec)
|
|
187
|
+
while (header = client.gets) && header != "\r\n"; end
|
|
195
188
|
end
|
|
196
189
|
end
|
|
197
190
|
|
|
191
|
+
unless request_line&.start_with?("GET /callback")
|
|
192
|
+
client.print "HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n"
|
|
193
|
+
return nil
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
params = parse_query_string(request_line)
|
|
197
|
+
code = params["code"]
|
|
198
|
+
error = params["error"]
|
|
199
|
+
|
|
200
|
+
if params["state"] != expected_state
|
|
201
|
+
client.print http_response(error_page("State mismatch. Please try again."))
|
|
202
|
+
nil
|
|
203
|
+
elsif error
|
|
204
|
+
client.print http_response(error_page(error))
|
|
205
|
+
raise AuthorizationDenied, error
|
|
206
|
+
elsif code.nil? || code.empty?
|
|
207
|
+
client.print http_response(error_page("No authorization code received."))
|
|
208
|
+
nil
|
|
209
|
+
else
|
|
210
|
+
client.print http_response(success_page)
|
|
211
|
+
code
|
|
212
|
+
end
|
|
213
|
+
rescue Timeout::Error, IOError, SystemCallError
|
|
198
214
|
nil
|
|
215
|
+
ensure
|
|
216
|
+
client.close
|
|
199
217
|
end
|
|
200
218
|
|
|
201
219
|
def parse_query_string(request_line)
|
|
@@ -205,6 +223,8 @@ module Flare
|
|
|
205
223
|
query = URI(path).query
|
|
206
224
|
return {} unless query
|
|
207
225
|
URI.decode_www_form(query).to_h
|
|
226
|
+
rescue URI::InvalidURIError, ArgumentError
|
|
227
|
+
{}
|
|
208
228
|
end
|
|
209
229
|
|
|
210
230
|
def http_response(body)
|
|
@@ -250,9 +270,12 @@ module Flare
|
|
|
250
270
|
puts " 3. Print token"
|
|
251
271
|
print "Choose (1/2/3): "
|
|
252
272
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
273
|
+
case $stdin.gets&.strip
|
|
274
|
+
when nil
|
|
275
|
+
# Closed stdin (a script, CI, a coding agent) can't answer. Asking again
|
|
276
|
+
# would never end and lose the token we just got, so show it.
|
|
277
|
+
puts
|
|
278
|
+
print_token(token)
|
|
256
279
|
when "1"
|
|
257
280
|
@saved_to_dotenv = true
|
|
258
281
|
save_to_dotenv(token)
|
|
@@ -354,7 +377,7 @@ module Flare
|
|
|
354
377
|
contents = File.read(gitignore_path)
|
|
355
378
|
entries_to_add = []
|
|
356
379
|
|
|
357
|
-
entries_to_add << ".env" if @saved_to_dotenv && !
|
|
380
|
+
entries_to_add << ".env" if (@saved_to_dotenv || key_in_dotenv?) && !env_ignored?(contents)
|
|
358
381
|
entries_to_add << "flare.sqlite3*" unless contents.include?("flare.sqlite3*")
|
|
359
382
|
|
|
360
383
|
return if entries_to_add.empty?
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "output"
|
|
4
|
+
require_relative "project"
|
|
4
5
|
require_relative "../version"
|
|
5
6
|
|
|
6
7
|
module Flare
|
|
7
8
|
class StatusCommand
|
|
8
9
|
include CLI::Output
|
|
10
|
+
include CLI::Project
|
|
9
11
|
|
|
10
12
|
def run
|
|
11
13
|
puts bold("Flare v#{VERSION}")
|
|
@@ -30,9 +32,10 @@ module Flare
|
|
|
30
32
|
if ENV["FLARE_KEY"] && !ENV["FLARE_KEY"].empty?
|
|
31
33
|
green("set via ENV")
|
|
32
34
|
else
|
|
33
|
-
|
|
34
|
-
if File.exist?(env_path) && File.read(env_path).match?(/^FLARE_KEY=.+/)
|
|
35
|
+
if key_in_dotenv?
|
|
35
36
|
green("set in .env")
|
|
37
|
+
elsif credentials_exist?
|
|
38
|
+
dim("not in ENV or .env (may be in Rails credentials)")
|
|
36
39
|
else
|
|
37
40
|
red("not configured")
|
|
38
41
|
end
|
data/lib/flare/cli.rb
CHANGED
|
@@ -8,6 +8,9 @@ module Flare
|
|
|
8
8
|
"setup" => "Authenticate and configure Flare for this project",
|
|
9
9
|
"doctor" => "Check your Flare setup for issues",
|
|
10
10
|
"status" => "Show current Flare configuration",
|
|
11
|
+
"requests" => "List recent requests from local development spans",
|
|
12
|
+
"request" => "Show one request's timeline from local development spans",
|
|
13
|
+
"nplus1" => "Find queries repeated within recent requests from local development spans",
|
|
11
14
|
"version" => "Print the Flare version",
|
|
12
15
|
"help" => "Show this help message",
|
|
13
16
|
}.freeze
|
|
@@ -26,6 +29,15 @@ module Flare
|
|
|
26
29
|
when "status"
|
|
27
30
|
require_relative "cli/status_command"
|
|
28
31
|
StatusCommand.new.run
|
|
32
|
+
when "requests"
|
|
33
|
+
require_relative "cli/requests_command"
|
|
34
|
+
RequestsCommand.new(argv.drop(1)).run
|
|
35
|
+
when "request"
|
|
36
|
+
require_relative "cli/request_command"
|
|
37
|
+
RequestCommand.new(argv.drop(1)).run
|
|
38
|
+
when "nplus1"
|
|
39
|
+
require_relative "cli/nplus1_command"
|
|
40
|
+
Nplus1Command.new(argv.drop(1)).run
|
|
29
41
|
when "version", "-v", "--version"
|
|
30
42
|
puts "flare #{Flare::VERSION}"
|
|
31
43
|
when "help", nil, "-h", "--help"
|
data/lib/flare/client_headers.rb
CHANGED
|
@@ -8,7 +8,8 @@ module Flare
|
|
|
8
8
|
# Single source of truth for the headers that identify this client and its
|
|
9
9
|
# version to the Flare API. The server parses the gem version out of the
|
|
10
10
|
# User-Agent ("Flare Ruby/X.Y.Z") to gate version-dependent features, and
|
|
11
|
-
# uses the X-Client-* headers to build a per-client picture.
|
|
11
|
+
# uses the X-Client-* headers to build a per-client picture. X-Client-Revision
|
|
12
|
+
# (when one resolves, see Configuration#revision) marks deploys on the charts.
|
|
12
13
|
#
|
|
13
14
|
# Safe to send on any request that targets the Flare API (metrics POST,
|
|
14
15
|
# rules GET, trace-notify POST). Do NOT send these on the presigned R2 blob
|
|
@@ -19,8 +20,8 @@ module Flare
|
|
|
19
20
|
module ClientHeaders
|
|
20
21
|
USER_AGENT = "Flare Ruby/#{Flare::VERSION}"
|
|
21
22
|
|
|
22
|
-
def self.to_h
|
|
23
|
-
{
|
|
23
|
+
def self.to_h(configuration = Flare.configuration)
|
|
24
|
+
headers = {
|
|
24
25
|
"User-Agent" => USER_AGENT,
|
|
25
26
|
"X-Client-Language" => "ruby",
|
|
26
27
|
"X-Client-Language-Version" => RUBY_VERSION,
|
|
@@ -28,6 +29,9 @@ module Flare
|
|
|
28
29
|
"X-Client-Pid" => Process.pid.to_s,
|
|
29
30
|
"X-Client-Hostname" => hostname
|
|
30
31
|
}
|
|
32
|
+
revision = configuration.revision
|
|
33
|
+
headers["X-Client-Revision"] = revision if revision
|
|
34
|
+
headers
|
|
31
35
|
end
|
|
32
36
|
|
|
33
37
|
def self.hostname
|
data/lib/flare/configuration.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "http_metrics_config"
|
|
4
|
+
require_relative "revision"
|
|
4
5
|
|
|
5
6
|
module Flare
|
|
6
7
|
class Configuration
|
|
@@ -50,6 +51,7 @@ module Flare
|
|
|
50
51
|
@ignore_request = ->(request) { false }
|
|
51
52
|
@subscribe_patterns = DEFAULT_SUBSCRIBE_PATTERNS.dup
|
|
52
53
|
@debug = ENV["FLARE_DEBUG"] == "1"
|
|
54
|
+
@revision = ENV["FLARE_REVISION"]
|
|
53
55
|
@http_metrics_config = HttpMetricsConfig::DEFAULT
|
|
54
56
|
|
|
55
57
|
# Environment-based defaults:
|
|
@@ -103,6 +105,24 @@ module Flare
|
|
|
103
105
|
yield @http_metrics_config
|
|
104
106
|
end
|
|
105
107
|
|
|
108
|
+
# The deployed revision, usually a git SHA, sent so Flare can mark deploys.
|
|
109
|
+
# Defaults to FLARE_REVISION; when neither is set, the first read asks
|
|
110
|
+
# Flare::Revision (platform variables, a REVISION file, then git).
|
|
111
|
+
def revision
|
|
112
|
+
return @revision if @revision_resolved
|
|
113
|
+
|
|
114
|
+
# Flag after the value, so a racing reader never sees the unresolved one.
|
|
115
|
+
# Two first readers may both resolve; they get the same answer.
|
|
116
|
+
@revision = Revision.resolve(@revision)
|
|
117
|
+
@revision_resolved = true
|
|
118
|
+
@revision
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def revision=(value)
|
|
122
|
+
@revision_resolved = false
|
|
123
|
+
@revision = value
|
|
124
|
+
end
|
|
125
|
+
|
|
106
126
|
private
|
|
107
127
|
|
|
108
128
|
def rails_development?
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_support"
|
|
4
|
+
require "active_support/parameter_filter"
|
|
5
|
+
require "set"
|
|
6
|
+
require "uri"
|
|
7
|
+
|
|
8
|
+
require_relative "sql_mask"
|
|
9
|
+
require_relative "store_statement"
|
|
10
|
+
require_relative "utf8"
|
|
11
|
+
|
|
12
|
+
module Flare
|
|
13
|
+
# Applies the app's filter_parameters and masks SQL values in span names and
|
|
14
|
+
# attributes before a trace leaves the process. Attributes that name the code
|
|
15
|
+
# or describe the request's shape are kept as they are, because the trace view
|
|
16
|
+
# and trace rules read them. Everything else is treated as possibly holding a
|
|
17
|
+
# value the app wouldn't want sent.
|
|
18
|
+
class ExportFilter
|
|
19
|
+
FILTERED = "[FILTERED]"
|
|
20
|
+
MAX_TEXT_BYTES = SqlMask::MAX_BYTES
|
|
21
|
+
|
|
22
|
+
# Applied even if the app's list is empty or was trimmed. Rails' generated
|
|
23
|
+
# list, plus the names credentials go by in URLs and headers.
|
|
24
|
+
DEFAULT_FILTERS = [
|
|
25
|
+
:passw, :email, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn, :cvv, :cvc,
|
|
26
|
+
/api.?key/i, /authoriz/i, /oauth/i, /\Aauth\z/i, /cookie/i, /session/i, /credential/i, /signature/i,
|
|
27
|
+
/jwt/i, /bearer/i, /pwd/i, /\Apass\z/i, /\Akey\z/i, /\Asig\z/i, /\Acode\z/i
|
|
28
|
+
].freeze
|
|
29
|
+
|
|
30
|
+
KEPT = %w[
|
|
31
|
+
name identifier layout count cache_hits record_count class_name
|
|
32
|
+
mailer action message_id hit exist store
|
|
33
|
+
http.method http.request.method http.status_code http.response.status_code
|
|
34
|
+
http.scheme url.scheme http.route http.user_agent user_agent.original
|
|
35
|
+
db.system db.system.name db.name db.namespace
|
|
36
|
+
db.sql.table db.collection.name db.redis.database_index
|
|
37
|
+
exception.type server.port net.peer.port
|
|
38
|
+
].to_set.freeze
|
|
39
|
+
KEPT_PREFIX = /\A(?:code|messaging|flare|otel)\./
|
|
40
|
+
|
|
41
|
+
# What these hold can't be sorted into safe and unsafe parts: message bodies
|
|
42
|
+
# and recipients, job arguments, request and exception contents.
|
|
43
|
+
OPAQUE = %w[
|
|
44
|
+
to from cc bcc reply_to subject mail
|
|
45
|
+
args arguments params parameters headers cookies body payload request response env session
|
|
46
|
+
exception exception_object exception.message exception.stacktrace
|
|
47
|
+
].to_set.freeze
|
|
48
|
+
OPAQUE_PREFIX = /\Aemail\./
|
|
49
|
+
|
|
50
|
+
OPERATIONS = %w[db.operation db.operation.name].to_set.freeze
|
|
51
|
+
STATEMENTS = %w[db.statement db.query.text db.query db.sql sql statement query].to_set.freeze
|
|
52
|
+
# A leading comment or parenthesis counts, since a statement can start with one.
|
|
53
|
+
SQL_TEXT = %r{\A[\s(]*(?:/\*|--|(?:select|insert|update|delete|replace|merge|with|set|create|alter|drop|truncate|grant|revoke|explain|describe|show|call|exec|execute|values|copy|pragma|vacuum|lock|begin|prepare)\b)}i
|
|
54
|
+
# With no system to go on, only statements that clearly read as queries get
|
|
55
|
+
# SQL masking, since it keeps bare words like the arguments of SET session:x y.
|
|
56
|
+
QUERY_TEXT = /\A[\s(]*(?:select|insert|update|delete|replace|merge|with|explain|values)\b/i
|
|
57
|
+
STRUCTURED_TEXT = /\A\s*[{\[]/
|
|
58
|
+
|
|
59
|
+
URLS = %w[http.url url.full url.path http.target http.referer url.original].to_set.freeze
|
|
60
|
+
HOSTS = %w[http.host server.address net.peer.name].to_set.freeze
|
|
61
|
+
USERINFO = /\A.*@/m
|
|
62
|
+
MONGO = /mongo/i
|
|
63
|
+
|
|
64
|
+
URL_START = %r{\A((?:[a-z][a-z0-9+.-]*:)?//)([^/?#]*)(.*)\z}im
|
|
65
|
+
NUMERIC_PORT = /\A(?:\[[^\]]*\]|[^:]*)(?::\d*)?\z/
|
|
66
|
+
SCHEMELESS_USERINFO = %r{\A[^/?#]+@}
|
|
67
|
+
PATH_USERINFO = %r{(?<=/|\A)[^/@]+@[^/]*}
|
|
68
|
+
|
|
69
|
+
# One scan over free text, leftmost first, so what a URL contains is never read
|
|
70
|
+
# as a key and value. key or quoted_key holds a pair's name.
|
|
71
|
+
TEXT_TOKEN = %r{
|
|
72
|
+
(?<url>(?<!\w)(?:(?<![.+\-])[a-z][a-z0-9+.-]*+:)?//[^\s'"<>]*+(?:['"](?>[^\s<>@]{0,256})@[^\s<>]*+)?)
|
|
73
|
+
|(?<auth>(?<!\w)(?:bearer\s+\S++|basic\s+[A-Za-z0-9+/=]{8,}))
|
|
74
|
+
|(?:
|
|
75
|
+
(?<!\\)\\*+(?<quote>["'])(?<quoted_key>[\w.\-\[\]%]++)\\*+\k<quote>\]?
|
|
76
|
+
|(?<![\w.\-\[\]%])(?<key>[\w.\-\[\]%]++)
|
|
77
|
+
)\s*+(?:=>|=|:)\s*+
|
|
78
|
+
}xi
|
|
79
|
+
# A value longer than this is masked rather than judged on part of it.
|
|
80
|
+
VALUE_WINDOW = 256
|
|
81
|
+
VALUE = /\A[^\s&;,]*+/
|
|
82
|
+
QUOTED_VALUE = /\A\\*+(?:["']|\z)/
|
|
83
|
+
NESTED_VALUE = %r{=|%3d|//|%2f%2f|@|%40}i
|
|
84
|
+
|
|
85
|
+
class << self
|
|
86
|
+
# Built on use, and rebuilt if the list changes, because initializers and
|
|
87
|
+
# engines can add to filter_parameters at any point during boot.
|
|
88
|
+
def current
|
|
89
|
+
filters = app_filters
|
|
90
|
+
cached = @current
|
|
91
|
+
return cached.last if cached && cached.first == filters
|
|
92
|
+
|
|
93
|
+
filter = new(filters)
|
|
94
|
+
@current = [filters.dup.freeze, filter]
|
|
95
|
+
filter
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
private
|
|
99
|
+
|
|
100
|
+
def app_filters
|
|
101
|
+
return [] unless defined?(Rails) && Rails.respond_to?(:application) && Rails.application
|
|
102
|
+
|
|
103
|
+
Array(Rails.application.config.filter_parameters)
|
|
104
|
+
rescue StandardError
|
|
105
|
+
[]
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def initialize(filters = [])
|
|
110
|
+
@parameter_filter = ActiveSupport::ParameterFilter.new(DEFAULT_FILTERS + filters)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def attributes(attrs)
|
|
114
|
+
return {} if attrs.nil?
|
|
115
|
+
|
|
116
|
+
system = attrs["db.system"] || attrs["db.system.name"]
|
|
117
|
+
attrs.each_with_object({}) { |(name, value), out| out[name] = attribute(UTF8.scrub(name.to_s), value, system) }
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def name(value)
|
|
121
|
+
return value unless value.is_a?(String)
|
|
122
|
+
|
|
123
|
+
free_text(clean(value))
|
|
124
|
+
rescue StandardError
|
|
125
|
+
FILTERED
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
private
|
|
129
|
+
|
|
130
|
+
def attribute(name, value, system)
|
|
131
|
+
return value if KEPT.include?(name) || KEPT_PREFIX.match?(name)
|
|
132
|
+
return FILTERED if OPAQUE.include?(name) || OPAQUE_PREFIX.match?(name)
|
|
133
|
+
|
|
134
|
+
case value
|
|
135
|
+
when String then text_attribute(name, value, system)
|
|
136
|
+
when Array then value.map { |item| attribute(name, item, system) }
|
|
137
|
+
else param_kept?(name, value) ? value : FILTERED
|
|
138
|
+
end
|
|
139
|
+
rescue StandardError
|
|
140
|
+
# A value that can't be processed doesn't go out.
|
|
141
|
+
FILTERED
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def text_attribute(name, value, system)
|
|
145
|
+
return statement(value, system) if STATEMENTS.include?(name)
|
|
146
|
+
return SqlMask.operation_word(value) || SqlMask::PLACEHOLDER if OPERATIONS.include?(name)
|
|
147
|
+
|
|
148
|
+
text = clean(value)
|
|
149
|
+
if URLS.include?(name) then url(text)
|
|
150
|
+
elsif HOSTS.include?(name) then text.sub(USERINFO, "")
|
|
151
|
+
elsif name == "url.query" then query_string(text)
|
|
152
|
+
elsif name == "key" then StoreStatement.masked_key(text)
|
|
153
|
+
elsif param_kept?(name, text) then free_text(text)
|
|
154
|
+
else FILTERED
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def clean(value)
|
|
159
|
+
UTF8.scrub(value.bytesize > MAX_TEXT_BYTES ? value.byteslice(0, MAX_TEXT_BYTES) : value)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def statement(text, system)
|
|
163
|
+
return StoreStatement.redis(text) if StoreStatement.redis_system?(system)
|
|
164
|
+
return StoreStatement.mongo(text) if MONGO.match?(system.to_s)
|
|
165
|
+
|
|
166
|
+
return command_only(clean(text)) unless SqlMask.sql_system?(system)
|
|
167
|
+
|
|
168
|
+
masked = SqlMask.mask(text, system)
|
|
169
|
+
return SqlMask::PLACEHOLDER if masked.empty?
|
|
170
|
+
return command_only(masked) if system.to_s.empty? && !QUERY_TEXT.match?(masked)
|
|
171
|
+
|
|
172
|
+
masked
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def command_only(text)
|
|
176
|
+
command = SqlMask.command_word(text)
|
|
177
|
+
command ? "#{command} #{SqlMask::PLACEHOLDER}" : SqlMask::PLACEHOLDER
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def url(text)
|
|
181
|
+
match = URL_START.match(text)
|
|
182
|
+
return without_fragment_and_query(text.sub(SCHEMELESS_USERINFO, "")) unless match
|
|
183
|
+
|
|
184
|
+
scheme, authority, rest = match.captures
|
|
185
|
+
if authority.include?("@")
|
|
186
|
+
authority = authority.sub(USERINFO, "")
|
|
187
|
+
elsif !NUMERIC_PORT.match?(authority)
|
|
188
|
+
return "#{scheme}#{FILTERED}"
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
"#{scheme}#{authority}#{without_fragment_and_query(rest)}"
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def without_fragment_and_query(text)
|
|
195
|
+
base, query = text.split("#", 2).first.to_s.split("?", 2)
|
|
196
|
+
base = base.to_s.gsub(PATH_USERINFO, FILTERED)
|
|
197
|
+
query ? "#{base}?#{query_string(query)}" : base
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def query_string(text)
|
|
201
|
+
text.gsub(/[^&;]+/) { |part| pair(part) }
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def pair(part)
|
|
205
|
+
key, value = part.split("=", 2)
|
|
206
|
+
return FILTERED if value.nil?
|
|
207
|
+
|
|
208
|
+
pair_kept?(key, value) ? part : "#{key}=#{FILTERED}"
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def pair_kept?(key, value)
|
|
212
|
+
!NESTED_VALUE.match?(value) && param_kept?(URI.decode_www_form_component(key), value)
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# user[password]=x is checked as nested params, so filters written as
|
|
216
|
+
# "user.password" apply to it too.
|
|
217
|
+
def param_kept?(key, value)
|
|
218
|
+
unless key.include?("[")
|
|
219
|
+
result = @parameter_filter.filter_param(key, value)
|
|
220
|
+
return result.equal?(value) || result == value
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
path = key.split(/[\[\]]+/).reject(&:empty?)
|
|
224
|
+
nested = path.reverse.inject(value) { |inner, part| { part => inner } }
|
|
225
|
+
@parameter_filter.filter(nested) == nested
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# Text with no known meaning: custom notification payloads, span names. Masks
|
|
229
|
+
# more than it needs to rather than guess where a value ends.
|
|
230
|
+
def free_text(text)
|
|
231
|
+
return FILTERED if STRUCTURED_TEXT.match?(text)
|
|
232
|
+
# What's left of SQL is still read for pairs, since a word like DROP can also
|
|
233
|
+
# start ordinary text.
|
|
234
|
+
text = SqlMask.mask(text) if SQL_TEXT.match?(text)
|
|
235
|
+
|
|
236
|
+
out = +""
|
|
237
|
+
position = 0
|
|
238
|
+
while (match = TEXT_TOKEN.match(text, position))
|
|
239
|
+
out << text[position...match.begin(0)]
|
|
240
|
+
position = match.end(0)
|
|
241
|
+
|
|
242
|
+
if match[:url] then out << text_url(match[:url])
|
|
243
|
+
elsif match[:auth] then out << match[:auth].split(/\s+/).first << " " << FILTERED
|
|
244
|
+
else
|
|
245
|
+
out << match[0]
|
|
246
|
+
value = text[position, VALUE_WINDOW + 1][VALUE]
|
|
247
|
+
next if value.length <= VALUE_WINDOW && pair_kept?(match[:key] || match[:quoted_key], value)
|
|
248
|
+
|
|
249
|
+
out << FILTERED
|
|
250
|
+
position = sensitive_value_end(text, position)
|
|
251
|
+
end
|
|
252
|
+
end
|
|
253
|
+
out << text[position..]
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
# A quote with an @ after it can't be read as a URL's parts, so none of it is kept.
|
|
257
|
+
def text_url(found)
|
|
258
|
+
scheme = found[/\A[^:\/]+(?=:)/]
|
|
259
|
+
return FILTERED if scheme && !param_kept?(scheme, found)
|
|
260
|
+
|
|
261
|
+
found.match?(/['"]/) ? "#{scheme}#{":" if scheme}//#{FILTERED}" : url(found)
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# A quoted value may hold escapes and line breaks, so it masks everything after
|
|
265
|
+
# it. Otherwise the line ends it, which covers multi-word values like cookie
|
|
266
|
+
# lists.
|
|
267
|
+
def sensitive_value_end(text, position)
|
|
268
|
+
return text.length if QUOTED_VALUE.match?(text[position, VALUE_WINDOW])
|
|
269
|
+
|
|
270
|
+
text.index("\n", position) || text.length
|
|
271
|
+
end
|
|
272
|
+
end
|
|
273
|
+
end
|
|
@@ -82,6 +82,15 @@ module Flare
|
|
|
82
82
|
complete: owner_finished || sampled_completion_span?(span_data),
|
|
83
83
|
delay: owner_finished ? @marked_trace_grace_period : 0
|
|
84
84
|
)
|
|
85
|
+
rescue StandardError => e
|
|
86
|
+
# The SDK calls processors without a rescue, so a bug here would raise
|
|
87
|
+
# into the app's request or job. Count every failure, warn once
|
|
88
|
+
# per process (keyed by pid so forked workers still warn).
|
|
89
|
+
@exception_count.increment
|
|
90
|
+
return if @on_finish_error_warned_pid == $$
|
|
91
|
+
|
|
92
|
+
@on_finish_error_warned_pid = $$
|
|
93
|
+
@logger.warn("[Flare::FilteringSpanProcessor] on_finish failed: #{e.class}: #{e.message}")
|
|
85
94
|
end
|
|
86
95
|
|
|
87
96
|
def force_flush(timeout: nil)
|
|
@@ -136,6 +145,8 @@ module Flare
|
|
|
136
145
|
|
|
137
146
|
def enqueue(span_data, complete:, delay: 0)
|
|
138
147
|
@mutex.synchronize do
|
|
148
|
+
return if @stopped
|
|
149
|
+
|
|
139
150
|
trace_id = span_data.trace_id
|
|
140
151
|
@trace_order << trace_id unless @pending_by_trace.key?(trace_id)
|
|
141
152
|
@pending_by_trace[trace_id] ||= []
|
|
@@ -323,6 +334,9 @@ module Flare
|
|
|
323
334
|
now = monotonic_now
|
|
324
335
|
ready_trace_ids = @delayed_ready_by_trace.select { |_, ready_at| ready_at <= now }.keys
|
|
325
336
|
ready_trace_ids.each do |trace_id|
|
|
337
|
+
# Overflow may have evicted the whole batch, so mark_trace_ready
|
|
338
|
+
# would not clear the entry and next_wait_timeout would return 0 forever.
|
|
339
|
+
@delayed_ready_by_trace.delete(trace_id)
|
|
326
340
|
mark_trace_ready(trace_id)
|
|
327
341
|
@marker.unmark(trace_id)
|
|
328
342
|
end
|
data/lib/flare/http_transport.rb
CHANGED
|
@@ -55,6 +55,10 @@ module Flare
|
|
|
55
55
|
http.open_timeout = effective_timeout(@open_timeout, deadline.remaining)
|
|
56
56
|
http.read_timeout = effective_timeout(@read_timeout, deadline.remaining)
|
|
57
57
|
http.write_timeout = effective_timeout(@write_timeout, deadline.remaining) if http.respond_to?(:write_timeout=)
|
|
58
|
+
# Net::HTTP silently resends a GET or PUT once after a read timeout or
|
|
59
|
+
# dropped connection, with a fresh timeout, so a slow server took twice
|
|
60
|
+
# the caller's budget. Callers decide when to try again.
|
|
61
|
+
http.max_retries = 0
|
|
58
62
|
|
|
59
63
|
req = klass.new(uri.request_uri == "" ? "/" : uri.request_uri)
|
|
60
64
|
headers.each { |k, v| req[k] = v }
|