packrat-rails 0.1.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +46 -4
- data/lib/generators/packrat/packrat_generator.rb +11 -3
- data/lib/generators/packrat/templates/initializer.rb.tt +7 -0
- data/lib/packrat/config.rb +50 -7
- data/lib/packrat/otel/error_spans.rb +16 -8
- data/lib/packrat/otel/scrubber.rb +221 -47
- data/lib/packrat/otel.rb +29 -7
- data/lib/packrat/version.rb +1 -1
- data/lib/packrat.rb +3 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ca86ba095ca40693f44e3bada4d7e6e71ca2635a37ffdcb1a512cb3362987c8c
|
|
4
|
+
data.tar.gz: e036105e4aa2eac0a688e6beb05fb44321b05ecb7388e12fdc5c0aca120dc70e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: '027381b226d874c0c6789aea42795698790e6dd995c8d67ba1551e3de79b9a283055dd8b67df38ce4ca57cc4e46cd827c3247be27e03870e412e9a33dc0afb31'
|
|
7
|
+
data.tar.gz: fbe00857f15e3aedc41db727dc22c5da574adddc1dfe648400fac289847f672c8f8707a50623d029b3250d08c8c477fa3fbed2346bfb2cfcd3d0bf59eb8b08ce
|
data/README.md
CHANGED
|
@@ -17,6 +17,8 @@ page:
|
|
|
17
17
|
```ruby
|
|
18
18
|
Packrat.configure do |config|
|
|
19
19
|
config.ingest_key = ENV.fetch("PACKRAT_INGEST_KEY", "packrat_ingest_…")
|
|
20
|
+
# :strict sends no log text, query strings or exception messages: https://packrat.sh/docs/ruby-on-rails#privacy
|
|
21
|
+
# config.privacy = :strict
|
|
20
22
|
end
|
|
21
23
|
```
|
|
22
24
|
|
|
@@ -25,20 +27,23 @@ run the generator without `--ingest-key` and set `PACKRAT_INGEST_KEY` where the
|
|
|
25
27
|
|
|
26
28
|
The generator also turns on the [Packrat plugin for Claude Code](https://packrat.sh/docs/claude-code-plugin)
|
|
27
29
|
in `.claude/settings.json`. Commit the file, and everyone who opens Claude Code in the repository and
|
|
28
|
-
trusts the folder gets the plugin
|
|
30
|
+
trusts the folder gets the plugin, and Claude Code keeps it up to date (`"autoUpdate": true` on the
|
|
31
|
+
`packrat` marketplace). If the file exists, the generator adds the plugin and keeps the rest.
|
|
29
32
|
To leave the file alone, pass `--no-claude-code`.
|
|
30
33
|
|
|
34
|
+
To start with [strict privacy](#privacy), pass `--strict`.
|
|
35
|
+
|
|
31
36
|
On boot the gem writes one line to the Rails log. It names the service and environment the app
|
|
32
37
|
reports under:
|
|
33
38
|
|
|
34
39
|
```
|
|
35
|
-
Packrat: sending traces and logs for shop/production to https://ingest.packrat.sh (rack, action_pack, active_job, active_record, net_http, pg, logger)
|
|
40
|
+
Packrat: sending traces and logs for shop/production to https://ingest.packrat.sh (rack, action_pack, active_job, active_record, net_http, pg, logger); redacting email and ip
|
|
36
41
|
```
|
|
37
42
|
|
|
38
43
|
## Configuration
|
|
39
44
|
|
|
40
|
-
The ingest key
|
|
41
|
-
variable.
|
|
45
|
+
The ingest key, the ingest URL and the privacy level are the only settings in `Packrat.configure`. Everything else is
|
|
46
|
+
an environment variable.
|
|
42
47
|
|
|
43
48
|
| Variable | Default | Note |
|
|
44
49
|
| ------------------------------- | --------------------------- | ------------------------------------------------------------ |
|
|
@@ -46,6 +51,8 @@ variable.
|
|
|
46
51
|
| `PACKRAT_INGEST_URL` | `config.ingest_url`, else `https://ingest.packrat.sh` | A local Packrat is `http://localhost:3100`. `--ingest-url` on the generator writes `config.ingest_url` |
|
|
47
52
|
| `PACKRAT_DISABLED` | unset | `1` turns the gem off in any environment |
|
|
48
53
|
| `PACKRAT_OTEL_INSTRUMENTATIONS` | the list below | Comma-separated names to add; a leading `-` removes one |
|
|
54
|
+
| `PACKRAT_PRIVACY` | `config.privacy`, else `standard` | `standard` or `strict`. Wins over the initializer. Any other value is strict |
|
|
55
|
+
| `PACKRAT_REDACT` | `email,ip` | Patterns redacted from values; `-ip` stops masking IP addresses |
|
|
49
56
|
|
|
50
57
|
The instrumentations are `rack`, `action_pack`, `action_view`, `active_job`, `active_record`,
|
|
51
58
|
`net_http`, `pg`, `mysql2`, `sidekiq`, `redis`, `aws_sdk`, `concurrent_ruby`, `rake` and `logger`. One for a library that is not in the app installs nothing.
|
|
@@ -60,6 +67,41 @@ app's module name (`Shop::Application` is `shop`). The environment is `Rails.env
|
|
|
60
67
|
|
|
61
68
|
The gem does nothing in the test environment.
|
|
62
69
|
|
|
70
|
+
## Privacy
|
|
71
|
+
|
|
72
|
+
The gem removes data inside your app's process, before anything is exported, so nothing it removes
|
|
73
|
+
reaches Packrat. There are two levels: `standard`, the default, and `strict`.
|
|
74
|
+
|
|
75
|
+
| Data | Standard | Strict |
|
|
76
|
+
| -------------------------------- | -------------------------------------------------------- | ---------------------------------------- |
|
|
77
|
+
| Request path, route and status | Sent | Sent |
|
|
78
|
+
| Query string | Sent, with filtered parameters replaced | Not sent |
|
|
79
|
+
| Request and response headers | Not sent | Not sent |
|
|
80
|
+
| User agent | Sent | Not sent |
|
|
81
|
+
| SQL and Redis commands | Sent with every value replaced by `?` | Sent with every value replaced by `?` |
|
|
82
|
+
| Log lines | Sent, redacted | Severity and request only; text is `[REDACTED]` |
|
|
83
|
+
| Exception class and stack frames | Sent | Sent |
|
|
84
|
+
| Exception message | Sent, redacted | `[REDACTED]`, in the stack as well |
|
|
85
|
+
| `Rails.error` context | Sent, redacted | Not sent |
|
|
86
|
+
| Job arguments | Not sent | Not sent |
|
|
87
|
+
| Client IP address | Not recorded on spans; masked in log lines | Not sent |
|
|
88
|
+
|
|
89
|
+
In standard, the gem redacts:
|
|
90
|
+
|
|
91
|
+
- **Email addresses**, which become `[email:4f2a91c0]`: the first eight hex digits of an HMAC-SHA256
|
|
92
|
+
keyed by the ingest key. The same address gives the same token, so you can still count per person.
|
|
93
|
+
- **IP addresses**, which keep their network: `203.0.113.77` becomes `203.0.113.0`, and IPv6 its
|
|
94
|
+
`/48`. The addresses of servers the app calls (`net.peer.ip`, `server.address`) are kept.
|
|
95
|
+
- **Parameters in `filter_parameters`**, in query strings and in `Rails.error` context: the same list
|
|
96
|
+
Rails filters your logs with.
|
|
97
|
+
- **Credentials**: values named like a key, token, password, secret, signature or credential, and
|
|
98
|
+
`Bearer` or `Basic` values.
|
|
99
|
+
|
|
100
|
+
Patterns can't recognize other personal data, such as a name or a support ticket's text in a log
|
|
101
|
+
line. If your logs can hold that, use strict: pass `--strict` to the generator, set
|
|
102
|
+
`config.privacy = :strict`, or set `PACKRAT_PRIVACY=strict`. Strict keeps routes, durations, status
|
|
103
|
+
codes, errors by class and the shape of each query, which is what a deploy check needs.
|
|
104
|
+
|
|
63
105
|
## Beside other tools
|
|
64
106
|
|
|
65
107
|
- **Nothing else:** the gem configures OpenTelemetry.
|
|
@@ -12,11 +12,14 @@ module Packrat
|
|
|
12
12
|
"and turns on the Packrat plugin for Claude Code in .claude/settings.json."
|
|
13
13
|
class_option :ingest_key, type: :string, desc: "The ingest key from your Packrat setup page"
|
|
14
14
|
class_option :ingest_url, type: :string, desc: "Where to send, when it isn't https://ingest.packrat.sh"
|
|
15
|
+
class_option :strict, type: :boolean, default: false,
|
|
16
|
+
desc: "Send no log text, query strings or exception messages"
|
|
15
17
|
class_option :claude_code, type: :boolean, default: true,
|
|
16
18
|
desc: "Turn on the Packrat plugin for Claude Code in .claude/settings.json"
|
|
17
19
|
|
|
18
20
|
CLAUDE_SETTINGS = ".claude/settings.json"
|
|
19
|
-
|
|
21
|
+
# Claude Code leaves a marketplace that is not Anthropic's off auto-update, and says nothing when it has a newer plugin.
|
|
22
|
+
MARKETPLACE = { "source" => { "source" => "github", "repo" => "packratlabs/claude-plugin" }, "autoUpdate" => true }.freeze
|
|
20
23
|
PLUGIN = "packrat@packrat"
|
|
21
24
|
|
|
22
25
|
def create_initializer
|
|
@@ -24,7 +27,7 @@ module Packrat
|
|
|
24
27
|
end
|
|
25
28
|
|
|
26
29
|
# Everyone who opens Claude Code in the repository and trusts the folder gets the plugin. Other
|
|
27
|
-
# settings in the file are kept, and so is a plugin someone turned off on purpose.
|
|
30
|
+
# settings in the file are kept, and so is a plugin someone turned off on purpose, or auto-update turned off.
|
|
28
31
|
def enable_claude_code_plugin
|
|
29
32
|
return unless options[:claude_code]
|
|
30
33
|
|
|
@@ -51,9 +54,14 @@ module Packrat
|
|
|
51
54
|
options[:ingest_url].to_s.strip
|
|
52
55
|
end
|
|
53
56
|
|
|
57
|
+
def strict?
|
|
58
|
+
options[:strict]
|
|
59
|
+
end
|
|
60
|
+
|
|
54
61
|
def settings_json(settings)
|
|
55
62
|
settings = settings.dup
|
|
56
|
-
|
|
63
|
+
known = settings["extraKnownMarketplaces"] || {}
|
|
64
|
+
settings["extraKnownMarketplaces"] = known.merge("packrat" => MARKETPLACE.merge(known["packrat"] || {}))
|
|
57
65
|
settings["enabledPlugins"] = { PLUGIN => true }.merge(settings["enabledPlugins"] || {})
|
|
58
66
|
"#{JSON.pretty_generate(settings)}\n"
|
|
59
67
|
end
|
|
@@ -6,6 +6,13 @@ Packrat.configure do |config|
|
|
|
6
6
|
<% else -%>
|
|
7
7
|
config.ingest_key = ENV.fetch("PACKRAT_INGEST_KEY", <%= ingest_key.inspect %>)
|
|
8
8
|
<% end -%>
|
|
9
|
+
<% if strict? -%>
|
|
10
|
+
# Strict sends no log text, query strings or exception messages: https://packrat.sh/docs/ruby-on-rails#privacy
|
|
11
|
+
config.privacy = :strict
|
|
12
|
+
<% else -%>
|
|
13
|
+
# :strict sends no log text, query strings or exception messages: https://packrat.sh/docs/ruby-on-rails#privacy
|
|
14
|
+
# config.privacy = :strict
|
|
15
|
+
<% end -%>
|
|
9
16
|
<% unless ingest_url.empty? -%>
|
|
10
17
|
config.ingest_url = <%= ingest_url.inspect %>
|
|
11
18
|
<% end -%>
|
data/lib/packrat/config.rb
CHANGED
|
@@ -8,11 +8,13 @@ module Packrat
|
|
|
8
8
|
DEFAULT_URL = "https://ingest.packrat.sh".freeze
|
|
9
9
|
INSTRUMENTATIONS = %w[rack action_pack action_view active_job active_record net_http pg mysql2 sidekiq redis aws_sdk concurrent_ruby rake logger].freeze
|
|
10
10
|
COMMIT_VARIABLES = %w[BUILD_COMMIT HEROKU_SLUG_COMMIT SOURCE_VERSION].freeze
|
|
11
|
+
REDACTIONS = %w[email ip].freeze
|
|
12
|
+
PRIVACY = %w[standard strict].freeze
|
|
11
13
|
REDACTED = "[redacted]".freeze
|
|
12
14
|
DISTRO_ATTRIBUTES = { "telemetry.distro.name" => "packrat-rails", "telemetry.distro.version" => VERSION }.freeze
|
|
13
15
|
|
|
14
|
-
attr_reader :rails_env, :app_name
|
|
15
|
-
attr_writer :ingest_key, :ingest_url
|
|
16
|
+
attr_reader :rails_env, :app_name, :redactor
|
|
17
|
+
attr_writer :ingest_key, :ingest_url, :privacy, :redact_keys, :redact_patterns, :debug_export
|
|
16
18
|
|
|
17
19
|
def initialize(env: ENV, rails_env: nil, app_name: nil)
|
|
18
20
|
@env = env
|
|
@@ -61,10 +63,45 @@ module Packrat
|
|
|
61
63
|
|
|
62
64
|
# PACKRAT_OTEL_INSTRUMENTATIONS=sidekiq,-pg adds sidekiq to the allowlist and takes pg out.
|
|
63
65
|
def instrumentations
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
66
|
+
adjusted(INSTRUMENTATIONS, "PACKRAT_OTEL_INSTRUMENTATIONS")
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# PACKRAT_REDACT=-ip stops masking IP addresses.
|
|
70
|
+
def redactions
|
|
71
|
+
adjusted(REDACTIONS, "PACKRAT_REDACT") & REDACTIONS
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# A value that is neither name is strict: a misspelling should not send more than was meant.
|
|
75
|
+
def privacy
|
|
76
|
+
value = (present(@env["PACKRAT_PRIVACY"]) || present(@privacy) || "standard").downcase
|
|
77
|
+
PRIVACY.include?(value) ? value : "strict"
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def strict?
|
|
81
|
+
privacy == "strict"
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Attribute and parameter names whose whole value is redacted: config.redact_keys = %w[ssn account_number].
|
|
85
|
+
def redact_keys
|
|
86
|
+
Array(@redact_keys).map { |name| name.to_s.strip }.reject(&:empty?).uniq
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# A string is matched literally, so a pattern written without regex syntax redacts what it says.
|
|
90
|
+
def redact_patterns
|
|
91
|
+
Array(@redact_patterns).filter_map do |pattern|
|
|
92
|
+
pattern.is_a?(Regexp) ? pattern : (Regexp.new(Regexp.escape(pattern.to_s)) unless pattern.to_s.empty?)
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# config.redact { |value, name| value.gsub(NAME, "[name]") } runs on every string after the gem's own
|
|
97
|
+
# redaction; name is the attribute's, or nil for a log line.
|
|
98
|
+
def redact(&block)
|
|
99
|
+
@redactor = block
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# The variable wins over the initializer, so PACKRAT_DEBUG_EXPORT=0 silences a committed setting.
|
|
103
|
+
def debug_export?
|
|
104
|
+
set?("PACKRAT_DEBUG_EXPORT") ? flag?("PACKRAT_DEBUG_EXPORT") : !!@debug_export
|
|
68
105
|
end
|
|
69
106
|
|
|
70
107
|
def service_name
|
|
@@ -108,7 +145,7 @@ module Packrat
|
|
|
108
145
|
end
|
|
109
146
|
|
|
110
147
|
def inspect
|
|
111
|
-
"#<#{self.class.name} url=#{url} key=#{key ? REDACTED : 'unset'} service=#{service_name} environment=#{environment}>"
|
|
148
|
+
"#<#{self.class.name} url=#{url} key=#{key ? REDACTED : 'unset'} service=#{service_name} environment=#{environment} privacy=#{privacy}>"
|
|
112
149
|
end
|
|
113
150
|
alias to_s inspect
|
|
114
151
|
|
|
@@ -122,6 +159,12 @@ module Packrat
|
|
|
122
159
|
end.to_h
|
|
123
160
|
end
|
|
124
161
|
|
|
162
|
+
def adjusted(defaults, name)
|
|
163
|
+
@env[name].to_s.split(",").map(&:strip).reject(&:empty?).reduce(defaults.dup) do |names, entry|
|
|
164
|
+
entry.start_with?("-") ? names - [ entry.delete_prefix("-") ] : names | [ entry ]
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
|
|
125
168
|
def sdk_disabled?
|
|
126
169
|
@env["OTEL_SDK_DISABLED"].to_s.strip.casecmp?("true")
|
|
127
170
|
end
|
|
@@ -14,11 +14,19 @@ module Packrat
|
|
|
14
14
|
# A cache failure falls back to the source of truth; Sentry skips these too.
|
|
15
15
|
CACHE_STORE_SOURCE = /_cache_store\.active_support\z/
|
|
16
16
|
|
|
17
|
+
def initialize(scrubber = Scrubber.new)
|
|
18
|
+
@scrubber = scrubber
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Scrubbed here as well as on export: with an app's own OpenTelemetry, its exporter receives
|
|
22
|
+
# these events too.
|
|
17
23
|
def report(error, handled:, severity:, context:, source: nil)
|
|
18
24
|
return if skipped?(handled, source)
|
|
19
25
|
|
|
20
26
|
exception = error.is_a?(Exception) ? error : RuntimeError.new(error.to_s)
|
|
21
|
-
attributes =
|
|
27
|
+
attributes = @scrubber.scrub_event_attributes(
|
|
28
|
+
exception_attributes(exception, handled, severity, source).merge(context_attributes(context))
|
|
29
|
+
)
|
|
22
30
|
span = OpenTelemetry::Trace.current_span
|
|
23
31
|
return span.record_exception(exception, attributes:) if span.recording?
|
|
24
32
|
|
|
@@ -35,11 +43,10 @@ module Packrat
|
|
|
35
43
|
(!handled && INSTRUMENTED_SOURCES.include?(source)) || CACHE_STORE_SOURCE.match?(source.to_s)
|
|
36
44
|
end
|
|
37
45
|
|
|
38
|
-
# Span events are not scrubbed on export, so the message and the stack are scrubbed here.
|
|
39
46
|
def exception_attributes(exception, handled, severity, source)
|
|
40
47
|
{
|
|
41
|
-
"exception.message" =>
|
|
42
|
-
"exception.stacktrace" =>
|
|
48
|
+
"exception.message" => utf8(exception.message),
|
|
49
|
+
"exception.stacktrace" => utf8(exception.full_message(highlight: false, order: :top)),
|
|
43
50
|
"exception.handled" => handled,
|
|
44
51
|
"rails.error.severity" => severity.to_s,
|
|
45
52
|
"rails.error.source" => source
|
|
@@ -58,13 +65,14 @@ module Packrat
|
|
|
58
65
|
def attribute_value(value)
|
|
59
66
|
case value
|
|
60
67
|
when Integer, Float, true, false then value
|
|
61
|
-
when Hash
|
|
62
|
-
|
|
68
|
+
when Hash then utf8(@scrubber.filter_hash(value).to_json)
|
|
69
|
+
when Array then utf8(value.to_json)
|
|
70
|
+
else utf8(value.to_s)
|
|
63
71
|
end
|
|
64
72
|
end
|
|
65
73
|
|
|
66
|
-
def
|
|
67
|
-
|
|
74
|
+
def utf8(text)
|
|
75
|
+
text.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "�")
|
|
68
76
|
end
|
|
69
77
|
end
|
|
70
78
|
end
|
|
@@ -1,16 +1,26 @@
|
|
|
1
1
|
require "cgi"
|
|
2
|
+
require "ipaddr"
|
|
3
|
+
require "json"
|
|
4
|
+
require "openssl"
|
|
5
|
+
require "active_support/parameter_filter"
|
|
2
6
|
|
|
3
7
|
module Packrat
|
|
4
8
|
module Otel
|
|
5
|
-
# Removes credentials before telemetry leaves the process. Packrat archives raw
|
|
6
|
-
# rewrites it, so anything that survives here is kept for good.
|
|
9
|
+
# Removes credentials and personal data before telemetry leaves the process. Packrat archives raw
|
|
10
|
+
# data and never rewrites it, so anything that survives here is kept for good.
|
|
7
11
|
#
|
|
8
|
-
# Query strings are kept on purpose: they carry what a request was about,
|
|
9
|
-
# a trace is read for. Only the values of credential-named parameters
|
|
10
|
-
|
|
12
|
+
# Query strings are kept on purpose unless privacy is strict: they carry what a request was about,
|
|
13
|
+
# which is most of what a trace is read for. Only the values of credential-named parameters, and of
|
|
14
|
+
# those the app's filter_parameters names, are replaced.
|
|
15
|
+
class Scrubber
|
|
11
16
|
REDACTED = "[REDACTED]".freeze
|
|
12
17
|
URL_KEYS = %w[http.target http.url url.full url.query].freeze
|
|
18
|
+
TEXT_KEYS = %w[exception.message exception.stacktrace].freeze
|
|
13
19
|
HEADER_PREFIXES = %w[http.request.header. http.response.header.].freeze
|
|
20
|
+
CONTEXT_PREFIX = "rails.error.context.".freeze
|
|
21
|
+
# The address of a server the app called, not of a person.
|
|
22
|
+
PEER_KEYS = %w[net.peer.ip net.peer.name network.peer.address server.address server.socket.address net.sock.peer.addr].freeze
|
|
23
|
+
STRICT_DROPPED_KEYS = %w[url.query http.user_agent user_agent.original].freeze
|
|
14
24
|
# A whole parameter name, or one segment of a snake_case name: api_key and access_token match,
|
|
15
25
|
# keyword and monkey do not.
|
|
16
26
|
CREDENTIAL_PARAM = /\A(?:[a-z0-9]+_)*(?:key|token|password|passwd|secret|signature|sig|auth|credential)(?:_[a-z0-9]+)*\z/i
|
|
@@ -22,75 +32,213 @@ module Packrat
|
|
|
22
32
|
/ix
|
|
23
33
|
BEARER = %r{\b(?<scheme>Bearer|Basic)\s+(?<value>[A-Za-z0-9._~+/-]+=*)}i
|
|
24
34
|
ANSI = /\e\[[0-9;]*m/
|
|
35
|
+
EMAIL = /[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}/
|
|
36
|
+
OCTET = /(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)/
|
|
37
|
+
# Not part of a longer dotted number, so a version such as 120.0.6099.109 is left alone.
|
|
38
|
+
IPV4 = /(?<![\d.])(#{OCTET}\.#{OCTET}\.#{OCTET})\.#{OCTET}(?!\.?\d)/
|
|
39
|
+
IPV6 = /(?<![\w:])(?:[0-9a-f]{0,4}:){2,7}[0-9a-f]{0,4}(?![\w:])/i
|
|
25
40
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
41
|
+
# keys, patterns and redactor are the app's own (config.redact_keys, redact_patterns, redact).
|
|
42
|
+
def initialize(key: nil, filters: [], redact: Config::REDACTIONS, strict: false, keys: [], patterns: [], redactor: nil)
|
|
43
|
+
@key = key.to_s
|
|
44
|
+
# A filter_parameters string matches any name that contains it; a key the app names matches whole.
|
|
45
|
+
filters = filters + keys.map { |name| /\A#{Regexp.escape(name)}\z/i }
|
|
46
|
+
@filter = ActiveSupport::ParameterFilter.new(filters, mask: REDACTED) unless filters.empty?
|
|
47
|
+
@keys = keys.map(&:downcase)
|
|
48
|
+
@patterns = patterns
|
|
49
|
+
@redactor = redactor
|
|
50
|
+
@email = redact.include?("email")
|
|
51
|
+
@ip = redact.include?("ip")
|
|
52
|
+
@strict = strict
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# The ingest key is the HMAC key, and is never printed.
|
|
56
|
+
def inspect
|
|
57
|
+
"#<#{self.class.name} email=#{@email} ip=#{@ip} strict=#{@strict} filters=#{!@filter.nil?} " \
|
|
58
|
+
"keys=#{@keys.size} patterns=#{@patterns.size} redactor=#{!@redactor.nil?}>"
|
|
59
|
+
end
|
|
60
|
+
alias to_s inspect
|
|
61
|
+
|
|
62
|
+
# A log body, an exception message or a stack.
|
|
63
|
+
def scrub_text(text)
|
|
64
|
+
return text unless text.is_a?(String)
|
|
65
|
+
|
|
66
|
+
text = text.gsub(CREDENTIAL_IN_TEXT) { "#{Regexp.last_match(:name)}#{REDACTED}" } if text.include?("=")
|
|
67
|
+
redact_values(text.gsub(BEARER) { "#{Regexp.last_match(:scheme)} #{REDACTED}" })
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# The same hash when nothing needs to change, so callers can tell by identity.
|
|
71
|
+
def scrub_attributes(attributes)
|
|
72
|
+
return attributes if attributes.nil? || attributes.empty?
|
|
29
73
|
|
|
30
|
-
|
|
31
|
-
|
|
74
|
+
scrubbed = attributes.each_with_object({}) do |(key, value), kept|
|
|
75
|
+
kept[key] = scrub_value(key.to_s, value) unless dropped?(key.to_s)
|
|
32
76
|
end
|
|
77
|
+
scrubbed == attributes ? attributes : scrubbed
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Strict keeps an exception's class and frames, and takes its message out of the stack as well.
|
|
81
|
+
def scrub_event_attributes(attributes)
|
|
82
|
+
message = attributes&.fetch("exception.message", nil)
|
|
83
|
+
return scrub_attributes(attributes) unless @strict && message.is_a?(String) && !message.empty?
|
|
84
|
+
|
|
85
|
+
stack = attributes["exception.stacktrace"]
|
|
86
|
+
stack = stack.gsub(message, REDACTED) if stack.is_a?(String)
|
|
87
|
+
scrub_attributes(attributes.merge("exception.message" => REDACTED, "exception.stacktrace" => stack).compact)
|
|
88
|
+
end
|
|
33
89
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
90
|
+
# A hash an app hands over whole, such as Rails.error's context, filtered as its logs would be.
|
|
91
|
+
def filter_hash(hash)
|
|
92
|
+
@filter ? @filter.filter(hash) : hash
|
|
93
|
+
end
|
|
38
94
|
|
|
39
|
-
|
|
40
|
-
|
|
95
|
+
# SpanData, Event and LogRecordData are frozen structs, so a changed one is rebuilt from its fields.
|
|
96
|
+
def scrub_spans(spans)
|
|
97
|
+
spans.map do |span|
|
|
98
|
+
attributes = scrub_attributes(span.attributes)
|
|
99
|
+
events = scrub_events(span.events)
|
|
100
|
+
next span if attributes.equal?(span.attributes) && events.equal?(span.events)
|
|
41
101
|
|
|
42
|
-
|
|
43
|
-
end
|
|
102
|
+
span.class.new(*span.to_h.merge(attributes:, events:).values)
|
|
44
103
|
end
|
|
104
|
+
end
|
|
45
105
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
106
|
+
def scrub_log_records(records)
|
|
107
|
+
records.map do |record|
|
|
108
|
+
attributes = scrub_attributes(record.attributes)
|
|
109
|
+
body = scrub_body(record.body)
|
|
110
|
+
next record if attributes.equal?(record.attributes) && body == record.body
|
|
111
|
+
|
|
112
|
+
record.class.new(*record.to_h.merge(attributes:, body:).values)
|
|
52
113
|
end
|
|
114
|
+
end
|
|
53
115
|
|
|
54
|
-
|
|
55
|
-
records.map do |record|
|
|
56
|
-
attributes = scrub_attributes(record.attributes)
|
|
57
|
-
body = record.body.is_a?(String) ? scrub_text(record.body.gsub(ANSI, "")) : record.body
|
|
58
|
-
next record if attributes.equal?(record.attributes) && body == record.body
|
|
116
|
+
private
|
|
59
117
|
|
|
60
|
-
|
|
61
|
-
|
|
118
|
+
def scrub_events(events)
|
|
119
|
+
return events if events.nil? || events.empty?
|
|
120
|
+
|
|
121
|
+
scrubbed = events.map do |event|
|
|
122
|
+
attributes = scrub_event_attributes(event.attributes)
|
|
123
|
+
attributes.equal?(event.attributes) ? event : event.class.new(*event.to_h.merge(attributes:).values)
|
|
62
124
|
end
|
|
125
|
+
scrubbed.zip(events).all? { |new, old| new.equal?(old) } ? events : scrubbed
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def scrub_body(body)
|
|
129
|
+
case body
|
|
130
|
+
when String then @strict ? REDACTED : custom(scrub_text(body.gsub(ANSI, "")), nil)
|
|
131
|
+
when Hash then filter_hash(body).transform_values { |value| scrub_body(value) }
|
|
132
|
+
when Array then body.map { |value| scrub_body(value) }
|
|
133
|
+
else body
|
|
134
|
+
end
|
|
135
|
+
end
|
|
63
136
|
|
|
64
|
-
|
|
137
|
+
# Fails closed: a value the app's block cannot answer for is not sent.
|
|
138
|
+
def custom(value, name)
|
|
139
|
+
return value unless @redactor && value.is_a?(String)
|
|
65
140
|
|
|
66
|
-
|
|
67
|
-
|
|
141
|
+
result = @redactor.call(value, name)
|
|
142
|
+
result.is_a?(String) ? result : REDACTED
|
|
143
|
+
rescue StandardError => error
|
|
144
|
+
unless @redactor_failed
|
|
145
|
+
@redactor_failed = true
|
|
146
|
+
OpenTelemetry.handle_error(exception: error, message: "Packrat: the config.redact block raised, so the values it was given are sent as #{REDACTED}")
|
|
68
147
|
end
|
|
148
|
+
REDACTED
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def dropped?(key)
|
|
152
|
+
HEADER_PREFIXES.any? { |prefix| key.start_with?(prefix) } ||
|
|
153
|
+
(@strict && (STRICT_DROPPED_KEYS.include?(key) || key.start_with?(CONTEXT_PREFIX)))
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def scrub_value(key, value)
|
|
157
|
+
return value.map { |item| scrub_value(key, item) } if value.is_a?(Array)
|
|
158
|
+
return value unless value.is_a?(String)
|
|
159
|
+
return REDACTED if redacted_key?(key)
|
|
160
|
+
|
|
161
|
+
custom(builtin(key, value), key)
|
|
162
|
+
end
|
|
69
163
|
|
|
70
|
-
|
|
71
|
-
|
|
164
|
+
def builtin(key, value)
|
|
165
|
+
if URL_KEYS.include?(key) then redact_values(@strict ? value.split("?", 2).first : redact_query(value))
|
|
166
|
+
elsif TEXT_KEYS.include?(key) then scrub_text(value)
|
|
167
|
+
elsif key.start_with?(CONTEXT_PREFIX) then filtered?(key.delete_prefix(CONTEXT_PREFIX)) ? REDACTED : scrub_text(value)
|
|
168
|
+
elsif PEER_KEYS.include?(key) then value
|
|
169
|
+
else redact_values(value)
|
|
72
170
|
end
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# The whole name, or its last dotted segment: ssn matches ssn and enduser.ssn.
|
|
174
|
+
def redacted_key?(key)
|
|
175
|
+
return false if @keys.empty?
|
|
176
|
+
|
|
177
|
+
name = key.downcase
|
|
178
|
+
@keys.include?(name) || @keys.include?(name.rpartition(".").last)
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def redact_values(text)
|
|
182
|
+
@patterns.each { |pattern| text = text.gsub(pattern, REDACTED) }
|
|
183
|
+
text = text.gsub(EMAIL) { "[email:#{digest(Regexp.last_match(0))}]" } if @email && text.include?("@")
|
|
184
|
+
return text unless @ip
|
|
185
|
+
|
|
186
|
+
text = text.gsub(IPV4) { "#{Regexp.last_match(1)}.0" } if text.match?(/\d\.\d/)
|
|
187
|
+
text = text.gsub(IPV6) { |match| ipv6(match) } if text.include?("::") || text.count(":") >= 7
|
|
188
|
+
text
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Keyed by the ingest key, so the same address is the same token within an app, and a list of
|
|
192
|
+
# addresses cannot be hashed to find one.
|
|
193
|
+
def digest(email)
|
|
194
|
+
OpenSSL::HMAC.hexdigest("SHA256", @key, email.downcase)[0, 8]
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# The /48 a network is usually given, as /24 is for IPv4. A clock time has colons too, so only a
|
|
198
|
+
# candidate IPAddr reads as IPv6 is masked.
|
|
199
|
+
def ipv6(candidate)
|
|
200
|
+
return candidate unless candidate.include?("::") || candidate.count(":") == 7
|
|
201
|
+
|
|
202
|
+
address = IPAddr.new(candidate)
|
|
203
|
+
address.ipv6? ? address.mask(48).to_s : candidate
|
|
204
|
+
rescue IPAddr::Error
|
|
205
|
+
candidate
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def redact_query(url)
|
|
209
|
+
return url unless url.include?("=")
|
|
73
210
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
return url if query.empty?
|
|
211
|
+
head, separator, query = url.include?("?") ? url.partition("?") : [ "", "", url ]
|
|
212
|
+
return url if query.empty?
|
|
77
213
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
"#{head}#{separator}#{redacted.join('&')}"
|
|
214
|
+
redacted = query.split("&").map do |pair|
|
|
215
|
+
name, equals, _value = pair.partition("=")
|
|
216
|
+
unescaped = CGI.unescape(name)
|
|
217
|
+
equals.empty? || !(CREDENTIAL_PARAM.match?(unescaped) || filtered?(unescaped)) ? pair : "#{name}=#{REDACTED}"
|
|
83
218
|
end
|
|
219
|
+
"#{head}#{separator}#{redacted.join('&')}"
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
def filtered?(name)
|
|
223
|
+
!@filter.nil? && @filter.filter(name => "")[name] == REDACTED
|
|
84
224
|
end
|
|
85
225
|
end
|
|
86
226
|
|
|
227
|
+
# With an io (PACKRAT_DEBUG_EXPORT), each item is also written there as one JSON line, exactly as
|
|
228
|
+
# scrubbed. Not through a Logger: the logger instrumentation would export each line, and print it again.
|
|
87
229
|
class ScrubbingSpanExporter
|
|
88
|
-
|
|
230
|
+
PREFIX = "Packrat export: ".freeze
|
|
231
|
+
|
|
232
|
+
def initialize(exporter, scrubber = Scrubber.new, io: nil)
|
|
89
233
|
@exporter = exporter
|
|
234
|
+
@scrubber = scrubber
|
|
235
|
+
@io = io
|
|
90
236
|
end
|
|
91
237
|
|
|
92
238
|
def export(spans, timeout: nil)
|
|
93
|
-
@
|
|
239
|
+
spans = @scrubber.scrub_spans(spans)
|
|
240
|
+
write_lines(spans) { |span| span_line(span) }
|
|
241
|
+
@exporter.export(spans, timeout:)
|
|
94
242
|
end
|
|
95
243
|
|
|
96
244
|
def force_flush(timeout: nil)
|
|
@@ -100,11 +248,37 @@ module Packrat
|
|
|
100
248
|
def shutdown(timeout: nil)
|
|
101
249
|
@exporter.shutdown(timeout:)
|
|
102
250
|
end
|
|
251
|
+
|
|
252
|
+
private
|
|
253
|
+
|
|
254
|
+
def write_lines(items)
|
|
255
|
+
return unless @io
|
|
256
|
+
|
|
257
|
+
@io.write(items.map { |item| "#{PREFIX}#{JSON.generate(yield(item))}\n" }.join)
|
|
258
|
+
rescue StandardError => error
|
|
259
|
+
OpenTelemetry.handle_error(exception: error, message: "Packrat: PACKRAT_DEBUG_EXPORT could not print a batch")
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
def span_line(span)
|
|
263
|
+
{
|
|
264
|
+
span: span.name, kind: span.kind, trace_id: span.hex_trace_id, span_id: span.hex_span_id,
|
|
265
|
+
attributes: span.attributes.to_h,
|
|
266
|
+
events: Array(span.events).map { |event| { name: event.name, attributes: event.attributes.to_h } }
|
|
267
|
+
}
|
|
268
|
+
end
|
|
103
269
|
end
|
|
104
270
|
|
|
105
271
|
class ScrubbingLogExporter < ScrubbingSpanExporter
|
|
106
272
|
def export(records, timeout: nil)
|
|
107
|
-
@
|
|
273
|
+
records = @scrubber.scrub_log_records(records)
|
|
274
|
+
write_lines(records) { |record| log_line(record) }
|
|
275
|
+
@exporter.export(records, timeout:)
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
private
|
|
279
|
+
|
|
280
|
+
def log_line(record)
|
|
281
|
+
{ log: record.body, severity: record.severity_text, attributes: record.attributes.to_h }
|
|
108
282
|
end
|
|
109
283
|
end
|
|
110
284
|
end
|
data/lib/packrat/otel.rb
CHANGED
|
@@ -32,10 +32,11 @@ module Packrat
|
|
|
32
32
|
class Setup
|
|
33
33
|
attr_reader :outcome, :line, :installed, :warnings
|
|
34
34
|
|
|
35
|
-
def initialize(config:, logger:, errors:)
|
|
35
|
+
def initialize(config:, logger:, errors:, filters: [])
|
|
36
36
|
@config = config
|
|
37
37
|
@logger = logger
|
|
38
38
|
@errors = errors
|
|
39
|
+
@filters = filters
|
|
39
40
|
@installed = []
|
|
40
41
|
@warnings = []
|
|
41
42
|
end
|
|
@@ -83,10 +84,10 @@ module Packrat
|
|
|
83
84
|
@outcome = :configured
|
|
84
85
|
step("sampler") { use_sampler } unless @config.set?("OTEL_TRACES_SAMPLER")
|
|
85
86
|
@config.instrumentations.each { |name| step("instrumentation #{name}") { install(name) } }
|
|
86
|
-
step("Rails.error") { @errors.subscribe(ErrorSpans.new) }
|
|
87
|
+
step("Rails.error") { @errors.subscribe(ErrorSpans.new(scrubber)) }
|
|
87
88
|
step("at_exit") { at_exit { shutdown_providers } }
|
|
88
89
|
@line = "Packrat: sending #{signals(records)} for #{@config.service_name}/#{@config.environment} " \
|
|
89
|
-
"to #{@config.url} (#{@installed.join(', ')})"
|
|
90
|
+
"to #{@config.url} (#{@installed.join(', ')}); #{privacy}"
|
|
90
91
|
end
|
|
91
92
|
|
|
92
93
|
def append(provider)
|
|
@@ -101,13 +102,13 @@ module Packrat
|
|
|
101
102
|
@outcome = :appended
|
|
102
103
|
records = sdk_logger_provider && logger_instrumented? && step("logs exporter") { log_record_processor }
|
|
103
104
|
records = nil unless records && step("logs processor") { sdk_logger_provider.add_log_record_processor(records) }
|
|
104
|
-
errors = !error_spans? && step("Rails.error") { @errors.subscribe(ErrorSpans.new) }
|
|
105
|
+
errors = !error_spans? && step("Rails.error") { @errors.subscribe(ErrorSpans.new(scrubber)) }
|
|
105
106
|
step("at_exit") { at_exit { [ spans, records ].compact.each { |processor| processor.shutdown(timeout: EXIT_TIMEOUT) } } }
|
|
106
107
|
|
|
107
108
|
attributes = provider.resource.attribute_enumerator.to_h
|
|
108
109
|
name = attributes["service.name"]
|
|
109
110
|
environment = attributes["deployment.environment.name"] || attributes["deployment.environment"]
|
|
110
|
-
@line = "Packrat: sending a copy of the app's #{signals(records)} for #{name}/#{environment} to #{@config.url}"
|
|
111
|
+
@line = "Packrat: sending a copy of the app's #{signals(records)} for #{name}/#{environment} to #{@config.url}; #{privacy}"
|
|
111
112
|
@line += "; Rails.error is now recorded on spans, which the app's own exporter also receives" if errors
|
|
112
113
|
if name.nil? || name.start_with?("unknown_service")
|
|
113
114
|
@line += ". Warning: the app's OpenTelemetry resource has no service.name, and the setup screen looks for the app by name"
|
|
@@ -140,13 +141,13 @@ module Packrat
|
|
|
140
141
|
|
|
141
142
|
def span_processor
|
|
142
143
|
OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
|
|
143
|
-
ScrubbingSpanExporter.new(OpenTelemetry::Exporter::OTLP::Exporter.new(endpoint: @config.traces_endpoint, headers: @config.headers))
|
|
144
|
+
ScrubbingSpanExporter.new(OpenTelemetry::Exporter::OTLP::Exporter.new(endpoint: @config.traces_endpoint, headers: @config.headers), scrubber, io: debug_io)
|
|
144
145
|
)
|
|
145
146
|
end
|
|
146
147
|
|
|
147
148
|
def log_record_processor
|
|
148
149
|
OpenTelemetry::SDK::Logs::Export::BatchLogRecordProcessor.new(
|
|
149
|
-
ScrubbingLogExporter.new(OpenTelemetry::Exporter::OTLP::Logs::LogsExporter.new(endpoint: @config.logs_endpoint, headers: @config.headers))
|
|
150
|
+
ScrubbingLogExporter.new(OpenTelemetry::Exporter::OTLP::Logs::LogsExporter.new(endpoint: @config.logs_endpoint, headers: @config.headers), scrubber, io: debug_io)
|
|
150
151
|
)
|
|
151
152
|
rescue NameError => error
|
|
152
153
|
raise unless error.name == :Utilities
|
|
@@ -242,6 +243,27 @@ module Packrat
|
|
|
242
243
|
Array(@errors.instance_variable_get(:@subscribers)).any? { |subscriber| subscriber.class.name.to_s.end_with?("ErrorSpans") }
|
|
243
244
|
end
|
|
244
245
|
|
|
246
|
+
def scrubber
|
|
247
|
+
@scrubber ||= Scrubber.new(
|
|
248
|
+
key: @config.key, filters: @filters, redact: @config.redactions, strict: @config.strict?,
|
|
249
|
+
keys: @config.redact_keys, patterns: @config.redact_patterns, redactor: @config.redactor
|
|
250
|
+
)
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
def debug_io
|
|
254
|
+
$stdout if @config.debug_export?
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
# The boot line says what is held back, so whoever reads the log knows without opening the README.
|
|
258
|
+
def privacy
|
|
259
|
+
redacted = @config.redactions.empty? ? "no personal data patterns" : @config.redactions.join(" and ")
|
|
260
|
+
own = { "keys" => @config.redact_keys, "patterns" => @config.redact_patterns, "block" => @config.redactor }
|
|
261
|
+
.filter_map { |name, value| (value.respond_to?(:empty?) ? !value.empty? : value) && name }
|
|
262
|
+
redacted += " and the app's own #{own.join(', ')}" unless own.empty?
|
|
263
|
+
line = @config.strict? ? "strict privacy, redacting #{redacted}" : "redacting #{redacted}"
|
|
264
|
+
@config.debug_export? ? "#{line}; printing every export to stdout (PACKRAT_DEBUG_EXPORT)" : line
|
|
265
|
+
end
|
|
266
|
+
|
|
245
267
|
def signals(records)
|
|
246
268
|
records ? "traces and logs" : "traces"
|
|
247
269
|
end
|
data/lib/packrat/version.rb
CHANGED
data/lib/packrat.rb
CHANGED
|
@@ -28,7 +28,9 @@ module Packrat
|
|
|
28
28
|
|
|
29
29
|
# An app adds to what the gem configured with ActiveSupport.on_load(:packrat) { |setup| ... }.
|
|
30
30
|
def start
|
|
31
|
-
@setup = Otel::Setup.new(
|
|
31
|
+
@setup = Otel::Setup.new(
|
|
32
|
+
config:, logger: Rails.logger, errors: Rails.error, filters: Rails.application.config.filter_parameters
|
|
33
|
+
).call
|
|
32
34
|
ActiveSupport.run_load_hooks(:packrat, @setup)
|
|
33
35
|
end
|
|
34
36
|
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: packrat-rails
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1
|
|
4
|
+
version: 0.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Packrat Labs
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-11-08 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: railties
|