instana 1.195.4 → 1.198.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +6 -2
  3. data/Rakefile +1 -1
  4. data/instana.gemspec +3 -7
  5. data/lib/instana.rb +3 -0
  6. data/lib/instana/activator.rb +2 -0
  7. data/lib/instana/backend/agent.rb +62 -0
  8. data/lib/instana/backend/gc_snapshot.rb +41 -0
  9. data/lib/instana/backend/host_agent.rb +74 -0
  10. data/lib/instana/backend/host_agent_activation_observer.rb +97 -0
  11. data/lib/instana/backend/host_agent_lookup.rb +57 -0
  12. data/lib/instana/backend/host_agent_reporting_observer.rb +106 -0
  13. data/lib/instana/backend/process_info.rb +64 -0
  14. data/lib/instana/backend/request_client.rb +74 -0
  15. data/lib/instana/backend/serverless_agent.rb +113 -0
  16. data/lib/instana/base.rb +10 -27
  17. data/lib/instana/config.rb +8 -22
  18. data/lib/instana/instrumentation/excon.rb +7 -5
  19. data/lib/instana/instrumentation/instrumented_request.rb +62 -7
  20. data/lib/instana/instrumentation/net-http.rb +7 -5
  21. data/lib/instana/instrumentation/rack.rb +12 -7
  22. data/lib/instana/logger_delegator.rb +31 -0
  23. data/lib/instana/{opentracing → open_tracing}/carrier.rb +0 -0
  24. data/lib/instana/open_tracing/instana_tracer.rb +99 -0
  25. data/lib/instana/secrets.rb +6 -2
  26. data/lib/instana/serverless.rb +139 -0
  27. data/lib/instana/setup.rb +23 -11
  28. data/lib/instana/snapshot/deltable.rb +25 -0
  29. data/lib/instana/snapshot/docker_container.rb +151 -0
  30. data/lib/instana/snapshot/fargate_container.rb +88 -0
  31. data/lib/instana/snapshot/fargate_process.rb +67 -0
  32. data/lib/instana/snapshot/fargate_task.rb +72 -0
  33. data/lib/instana/snapshot/lambda_function.rb +39 -0
  34. data/lib/instana/snapshot/ruby_process.rb +48 -0
  35. data/lib/instana/tracer.rb +25 -143
  36. data/lib/instana/tracing/processor.rb +14 -22
  37. data/lib/instana/tracing/span.rb +33 -36
  38. data/lib/instana/tracing/span_context.rb +15 -10
  39. data/lib/instana/util.rb +8 -69
  40. data/lib/instana/version.rb +1 -1
  41. data/lib/opentracing.rb +26 -3
  42. data/test/backend/agent_test.rb +67 -0
  43. data/test/backend/gc_snapshot_test.rb +11 -0
  44. data/test/backend/host_agent_activation_observer_test.rb +72 -0
  45. data/test/backend/host_agent_lookup_test.rb +78 -0
  46. data/test/backend/host_agent_reporting_observer_test.rb +192 -0
  47. data/test/backend/host_agent_test.rb +47 -0
  48. data/test/backend/process_info_test.rb +63 -0
  49. data/test/backend/request_client_test.rb +39 -0
  50. data/test/backend/serverless_agent_test.rb +73 -0
  51. data/test/config_test.rb +10 -0
  52. data/test/instana_test.rb +11 -4
  53. data/test/instrumentation/rack_instrumented_request_test.rb +5 -2
  54. data/test/instrumentation/rack_test.rb +2 -14
  55. data/test/secrets_test.rb +41 -22
  56. data/test/serverless_test.rb +323 -0
  57. data/test/snapshot/deltable_test.rb +17 -0
  58. data/test/snapshot/docker_container_test.rb +82 -0
  59. data/test/snapshot/fargate_container_test.rb +82 -0
  60. data/test/snapshot/fargate_process_test.rb +35 -0
  61. data/test/snapshot/fargate_task_test.rb +49 -0
  62. data/test/snapshot/lambda_function_test.rb +37 -0
  63. data/test/snapshot/ruby_process_test.rb +14 -0
  64. data/test/support/mock_timer.rb +20 -0
  65. data/test/test_helper.rb +16 -4
  66. data/test/tracing/custom_test.rb +1 -3
  67. data/test/tracing/id_management_test.rb +8 -0
  68. data/test/tracing/opentracing_test.rb +15 -2
  69. data/test/tracing/processor_test.rb +58 -0
  70. data/test/tracing/span_context_test.rb +22 -0
  71. data/test/tracing/span_test.rb +136 -0
  72. data/test/tracing/tracer_async_test.rb +29 -0
  73. data/test/tracing/tracer_test.rb +82 -16
  74. data/test/util_test.rb +10 -0
  75. metadata +76 -43
  76. data/lib/instana/agent.rb +0 -508
  77. data/lib/instana/agent/helpers.rb +0 -87
  78. data/lib/instana/agent/hooks.rb +0 -44
  79. data/lib/instana/agent/tasks.rb +0 -51
  80. data/lib/instana/collector.rb +0 -119
  81. data/lib/instana/collectors/gc.rb +0 -60
  82. data/lib/instana/collectors/memory.rb +0 -37
  83. data/lib/instana/collectors/thread.rb +0 -33
  84. data/lib/instana/eum/eum-test.js.erb +0 -17
  85. data/lib/instana/eum/eum.js.erb +0 -17
  86. data/lib/instana/helpers.rb +0 -47
  87. data/lib/instana/opentracing/tracer.rb +0 -21
  88. data/lib/instana/thread_local.rb +0 -18
  89. data/lib/oj_check.rb +0 -19
  90. data/test/agent/agent_test.rb +0 -151
@@ -0,0 +1,47 @@
1
+ # (c) Copyright IBM Corp. 2021
2
+ # (c) Copyright Instana Inc. 2021
3
+
4
+ require 'test_helper'
5
+
6
+ class HostAgentTest < Minitest::Test
7
+ def test_spawn_background_thread
8
+ ENV['INSTANA_TEST'] = nil
9
+ ::Instana.config[:agent_host] = '10.10.10.10'
10
+
11
+ if File.exist?('/sbin/ip')
12
+ addr = `/sbin/ip route | awk '/default/ { print $3 }'`.strip
13
+ stub_request(:get, "http://#{addr}:42699/")
14
+ .to_timeout
15
+ end
16
+
17
+ stub_request(:get, "http://10.10.10.10:42699/")
18
+ .to_timeout.times(3).then
19
+ .to_return(status: 200, body: "", headers: {})
20
+
21
+ discovery = Minitest::Mock.new
22
+ discovery.expect(:delete_observers, discovery, [])
23
+ discovery.expect(:with_observer, discovery, [Instana::Backend::HostAgentActivationObserver])
24
+ discovery.expect(:with_observer, discovery, [Instana::Backend::HostAgentReportingObserver])
25
+ discovery.expect(:swap, discovery, [])
26
+
27
+ subject = Instana::Backend::HostAgent.new(discovery: discovery)
28
+
29
+ FakeFS.with_fresh do
30
+ FakeFS::FileSystem.clone('test/support/ecs', '/proc')
31
+ subject.spawn_background_thread
32
+ end
33
+
34
+ subject.future.value!
35
+
36
+ discovery.verify
37
+ ensure
38
+ ::Instana.config[:agent_host] = '127.0.0.1'
39
+ ENV['INSTANA_TEST'] = 'true'
40
+ end
41
+
42
+ def test_discovery_value
43
+ discovery = Concurrent::Atom.new({'pid' => 1})
44
+ subject = Instana::Backend::HostAgent.new(discovery: discovery)
45
+ assert_equal 1, subject.source[:e]
46
+ end
47
+ end
@@ -0,0 +1,63 @@
1
+ # (c) Copyright IBM Corp. 2021
2
+ # (c) Copyright Instana Inc. 2021
3
+
4
+ require 'test_helper'
5
+
6
+ class ProcessInfoTest < Minitest::Test
7
+ def test_osx_argument_stripping
8
+ host_os = RbConfig::CONFIG['host_os']
9
+ RbConfig::CONFIG['host_os'] = 'darwin'
10
+
11
+ subject = Instana::Backend::ProcessInfo.new(OpenStruct.new(cmdline: 'test INSTANA_TEST=1 KV=2'))
12
+ assert_equal ['KV=2'], subject.arguments
13
+ ensure
14
+ RbConfig::CONFIG['host_os'] = host_os
15
+ end
16
+
17
+ def test_linux_argument_stripping
18
+ host_os = RbConfig::CONFIG['host_os']
19
+ RbConfig::CONFIG['host_os'] = 'linux'
20
+
21
+ subject = Instana::Backend::ProcessInfo.new(OpenStruct.new(cmdline: 'test INSTANA_TEST=1 KV=2'))
22
+ assert_equal ['INSTANA_TEST=1', 'KV=2'], subject.arguments
23
+ ensure
24
+ RbConfig::CONFIG['host_os'] = host_os
25
+ end
26
+
27
+ def test_no_proc
28
+ subject = Instana::Backend::ProcessInfo.new(OpenStruct.new(pid: 0))
29
+
30
+ FakeFS.with_fresh do
31
+ FakeFS::FileSystem.clone('test/support/ecs', '/proc')
32
+
33
+ assert_equal 0, subject.parent_pid
34
+ assert_nil subject.cpuset
35
+ assert_nil subject.sched_pid
36
+ refute subject.in_container?
37
+ end
38
+ end
39
+
40
+ def test_cpuset_proc
41
+ subject = Instana::Backend::ProcessInfo.new(OpenStruct.new(pid: 0))
42
+
43
+ FakeFS do
44
+ FakeFS::FileSystem.clone('test/support/proc', '/proc')
45
+ assert_equal '/', subject.cpuset
46
+ refute subject.in_container?
47
+ end
48
+ end
49
+
50
+ def test_sched_pid
51
+ subject = Instana::Backend::ProcessInfo.new(OpenStruct.new(pid: 1))
52
+
53
+ FakeFS do
54
+ FakeFS::FileSystem.clone('test/support/proc', '/proc')
55
+ refute_equal '/', subject.cpuset
56
+
57
+ assert subject.in_container?
58
+ assert_equal 35, subject.sched_pid
59
+ assert subject.from_parent_namespace
60
+ assert_equal subject.sched_pid, subject.parent_pid
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,39 @@
1
+ # (c) Copyright IBM Corp. 2021
2
+ # (c) Copyright Instana Inc. 2021
3
+
4
+ require 'test_helper'
5
+
6
+ class RequestClientTest < Minitest::Test
7
+ def test_send_request_simple
8
+ stub_request(:get, 'http://example.com:9292/')
9
+ .to_return(body: 'ok', status: '200')
10
+
11
+ subject = Instana::Backend::RequestClient.new('example.com', 9292)
12
+ response = subject.send_request('GET', '/')
13
+
14
+ assert response.ok?
15
+ assert 'ok', response.body
16
+ end
17
+
18
+ def test_send_request_json
19
+ stub_request(:post, 'http://example.com:9292/')
20
+ .with(body: '{"key":"value"}')
21
+ .to_return(body: '{"ok": true}', status: '200')
22
+
23
+ subject = Instana::Backend::RequestClient.new('example.com', 9292)
24
+ response = subject.send_request('POST', '/', {key: 'value'})
25
+
26
+ assert response.ok?
27
+ assert_equal({"ok" => true}, response.json)
28
+ end
29
+
30
+ def test_send_request_failure
31
+ stub_request(:get, 'http://example.com:9292/')
32
+ .to_return(status: '500')
33
+
34
+ subject = Instana::Backend::RequestClient.new('example.com', 9292)
35
+ response = subject.send_request('GET', '/')
36
+
37
+ refute response.ok?
38
+ end
39
+ end
@@ -0,0 +1,73 @@
1
+ # (c) Copyright IBM Corp. 2021
2
+ # (c) Copyright Instana Inc. 2021
3
+
4
+ require 'test_helper'
5
+
6
+ class ServerlesAgentTest < Minitest::Test
7
+ def test_report
8
+ stub_request(:post, "http://10.10.10.10:9292//bundle")
9
+ .to_return(status: 500)
10
+ .to_return(status: 200)
11
+
12
+ host_name = Class.new do
13
+ def host_name
14
+ 'hello'
15
+ end
16
+ end.new
17
+
18
+ snapshots = [Instana::Snapshot::RubyProcess.new, host_name]
19
+ subject = Instana::Backend::ServerlessAgent.new(snapshots, timer_class: MockTimer, backend_uri: 'http://10.10.10.10:9292/', logger: Logger.new('/dev/null'))
20
+
21
+ subject.timer.block.call
22
+ subject.timer.block.call
23
+ end
24
+
25
+ def test_ready
26
+ subject = Instana::Backend::ServerlessAgent.new([], timer_class: MockTimer, backend_uri: 'http://10.10.10.10:9292/')
27
+ assert subject.ready?
28
+ end
29
+
30
+ def test_extra_headers
31
+ subject = Instana::Backend::ServerlessAgent.new([], timer_class: MockTimer, backend_uri: 'http://10.10.10.10:9292/')
32
+ assert_equal [], subject.extra_headers
33
+ end
34
+
35
+ def test_secret_values
36
+ subject = Instana::Backend::ServerlessAgent.new([], timer_class: MockTimer, backend_uri: 'http://10.10.10.10:9292/')
37
+ assert_equal({"matcher" => "contains-ignore-case", "list" => %w[key password secret]}, subject.secret_values)
38
+ end
39
+
40
+ def test_spawn_background_thread
41
+ subject = Instana::Backend::ServerlessAgent.new([], timer_class: MockTimer, backend_uri: 'http://10.10.10.10:9292/')
42
+ subject.spawn_background_thread
43
+
44
+ assert subject.timer.running
45
+ end
46
+
47
+ def test_source
48
+ snapshot = Class.new do
49
+ def source
50
+ {test: 1}
51
+ end
52
+ end.new
53
+ subject = Instana::Backend::ServerlessAgent.new([snapshot], timer_class: MockTimer, backend_uri: 'http://10.10.10.10:9292/')
54
+
55
+ assert_equal({test: 1}, subject.source)
56
+ assert_equal({test: 1}, subject.source)
57
+ end
58
+
59
+ def test_missing_source
60
+ subject = Instana::Backend::ServerlessAgent.new([], timer_class: MockTimer, backend_uri: 'http://10.10.10.10:9292/', logger: Logger.new('/dev/null'))
61
+
62
+ assert_equal({}, subject.source)
63
+ end
64
+
65
+ def test_report_error
66
+ stub_request(:post, "http://10.10.10.10:9292//bundle")
67
+ .to_return(status: 500)
68
+
69
+ subject = Instana::Backend::ServerlessAgent.new([], timer_class: MockTimer, backend_uri: 'http://10.10.10.10:9292/', logger: Logger.new('/dev/null'))
70
+
71
+ subject.timer.block.call
72
+ end
73
+ end
data/test/config_test.rb CHANGED
@@ -21,4 +21,14 @@ class ConfigTest < Minitest::Test
21
21
  assert_equal true, ::Instana.config[:metrics][k].key?(:enabled)
22
22
  end
23
23
  end
24
+
25
+ def test_custom_agent_host
26
+ subject = Instana::Config.new(logger: Logger.new('/dev/null'), agent_host: 'abc')
27
+ assert_equal 'abc', subject[:agent_host]
28
+ end
29
+
30
+ def test_custom_agent_port
31
+ subject = Instana::Config.new(logger: Logger.new('/dev/null'), agent_port: 'abc')
32
+ assert_equal 'abc', subject[:agent_port]
33
+ end
24
34
  end
data/test/instana_test.rb CHANGED
@@ -16,10 +16,6 @@ class InstanaTest < Minitest::Test
16
16
  refute_nil ::Instana.agent
17
17
  end
18
18
 
19
- def test_that_it_has_a_collector
20
- refute_nil ::Instana.collector
21
- end
22
-
23
19
  def test_that_it_has_a_tracer
24
20
  refute_nil ::Instana.tracer
25
21
  end
@@ -27,4 +23,15 @@ class InstanaTest < Minitest::Test
27
23
  def test_that_it_has_a_config
28
24
  refute_nil ::Instana.config
29
25
  end
26
+
27
+ def test_assign_logger
28
+ mock = Minitest::Mock.new
29
+ mock.expect(:info, true, [String])
30
+
31
+ ::Instana.logger = mock
32
+ ::Instana.logger.info('test')
33
+ ::Instana.logger = Logger.new('/dev/null')
34
+
35
+ mock.verify
36
+ end
30
37
  end
@@ -29,6 +29,7 @@ class RackInstrumentedRequestTest < Minitest::Test
29
29
  expected = {
30
30
  trace_id: id,
31
31
  span_id: id,
32
+ from_w3: false,
32
33
  level: '1'
33
34
  }
34
35
 
@@ -47,6 +48,7 @@ class RackInstrumentedRequestTest < Minitest::Test
47
48
  external_state: nil,
48
49
  trace_id: 'a3ce929d0e0e4736',
49
50
  span_id: '00f067aa0ba902b7',
51
+ from_w3: true,
50
52
  level: '1'
51
53
  }
52
54
 
@@ -81,7 +83,8 @@ class RackInstrumentedRequestTest < Minitest::Test
81
83
  end
82
84
 
83
85
  def test_request_tags
84
- ::Instana.agent.extra_headers = %w[X-Capture-This]
86
+ ::Instana.agent.define_singleton_method(:extra_headers) { %w[X-Capture-This] }
87
+
85
88
  req = Instana::InstrumentedRequest.new(
86
89
  'HTTP_HOST' => 'example.com',
87
90
  'REQUEST_METHOD' => 'GET',
@@ -101,7 +104,7 @@ class RackInstrumentedRequestTest < Minitest::Test
101
104
  }
102
105
 
103
106
  assert_equal expected, req.request_tags
104
- ::Instana.agent.extra_headers = nil
107
+ ::Instana.agent.singleton_class.send(:remove_method, :extra_headers)
105
108
  end
106
109
 
107
110
  def test_correlation_data_valid
@@ -80,9 +80,6 @@ class RackTest < Minitest::Test
80
80
  assert_equal 200, rack_span[:data][:http][:status]
81
81
  assert_equal 'example.org', rack_span[:data][:http][:host]
82
82
  assert rack_span.key?(:f)
83
- assert rack_span[:f].key?(:e)
84
- assert rack_span[:f].key?(:h)
85
- assert_equal ::Instana.agent.agent_uuid, rack_span[:f][:h]
86
83
  assert !rack_span.key?(:stack)
87
84
 
88
85
  # Restore to default
@@ -134,9 +131,6 @@ class RackTest < Minitest::Test
134
131
  assert_equal "/mrlobster", rack_span[:data][:http][:url]
135
132
  assert_equal 200, rack_span[:data][:http][:status]
136
133
  assert rack_span.key?(:f)
137
- assert rack_span[:f].key?(:e)
138
- assert rack_span[:f].key?(:h)
139
- assert_equal ::Instana.agent.agent_uuid, rack_span[:f][:h]
140
134
  end
141
135
 
142
136
  def test_basic_put
@@ -166,9 +160,6 @@ class RackTest < Minitest::Test
166
160
  assert_equal "/mrlobster", rack_span[:data][:http][:url]
167
161
  assert_equal 200, rack_span[:data][:http][:status]
168
162
  assert rack_span.key?(:f)
169
- assert rack_span[:f].key?(:e)
170
- assert rack_span[:f].key?(:h)
171
- assert_equal ::Instana.agent.agent_uuid, rack_span[:f][:h]
172
163
  end
173
164
 
174
165
  def test_context_continuation
@@ -202,9 +193,6 @@ class RackTest < Minitest::Test
202
193
  assert_equal "/mrlobster", rack_span[:data][:http][:url]
203
194
  assert_equal 200, rack_span[:data][:http][:status]
204
195
  assert rack_span.key?(:f)
205
- assert rack_span[:f].key?(:e)
206
- assert rack_span[:f].key?(:h)
207
- assert_equal ::Instana.agent.agent_uuid, rack_span[:f][:h]
208
196
 
209
197
  # Context validation
210
198
  # The first span should have the passed in trace ID
@@ -260,7 +248,7 @@ class RackTest < Minitest::Test
260
248
  def test_custom_headers_capture
261
249
  clear_all!
262
250
  ::Instana.config[:collect_backtraces] = true
263
- ::Instana.agent.extra_headers = %w(X-Capture-This X-Capture-That)
251
+ ::Instana.agent.define_singleton_method(:extra_headers) { %w(X-Capture-This X-Capture-That) }
264
252
 
265
253
  get '/mrlobster', {}, { "HTTP_X_CAPTURE_THIS" => "ThereYouGo" }
266
254
  assert last_response.ok?
@@ -280,7 +268,7 @@ class RackTest < Minitest::Test
280
268
 
281
269
  # Restore to default
282
270
  ::Instana.config[:collect_backtraces] = false
283
- ::Instana.agent.extra_headers = nil
271
+ ::Instana.agent.singleton_class.send(:remove_method, :extra_headers)
284
272
  end
285
273
 
286
274
  def test_capture_http_path_template
data/test/secrets_test.rb CHANGED
@@ -5,72 +5,91 @@ require 'test_helper'
5
5
 
6
6
  class SecretsTest < Minitest::Test
7
7
  def setup
8
- @subject = Instana::Secrets.new
8
+ @subject = Instana::Secrets.new(logger: Logger.new('/dev/null'))
9
9
  end
10
-
10
+
11
11
  def test_equals_ignore_case
12
12
  sample_config = {
13
- "matcher"=>"equals-ignore-case",
13
+ "matcher"=>"equals-ignore-case",
14
14
  "list"=>["key"]
15
15
  }
16
-
16
+
17
17
  url = url_for(%w(key Str kEy KEY))
18
18
  assert_redacted @subject.remove_from_query(url, sample_config), %w(key kEy KEY)
19
19
  end
20
-
20
+
21
21
  def test_equals
22
22
  sample_config = {
23
- "matcher"=>"equals",
23
+ "matcher"=>"equals",
24
24
  "list"=>["key", "kEy"]
25
25
  }
26
-
26
+
27
27
  url = url_for(%w(key Str kEy KEY))
28
28
  assert_redacted @subject.remove_from_query(url, sample_config), %w(key kEy)
29
29
  end
30
-
30
+
31
31
  def test_contains_ignore_case
32
32
  sample_config = {
33
- "matcher"=>"contains-ignore-case",
33
+ "matcher"=>"contains-ignore-case",
34
34
  "list"=>["stan"]
35
35
  }
36
-
36
+
37
37
  url = url_for(%w(instantiate conTESTant sample))
38
38
  assert_redacted @subject.remove_from_query(url, sample_config), %w(instantiate conTESTant)
39
39
  end
40
-
40
+
41
41
  def test_contains
42
42
  sample_config = {
43
- "matcher"=>"contains",
43
+ "matcher"=>"contains",
44
44
  "list"=>["stan"]
45
45
  }
46
-
46
+
47
47
  url = url_for(%w(instantiate conTESTant sample))
48
48
  assert_redacted @subject.remove_from_query(url, sample_config), %w(instantiate)
49
-
50
49
  end
51
-
50
+
52
51
  def test_regexp
53
52
  sample_config = {
54
- "matcher"=>"regex",
53
+ "matcher"=>"regex",
55
54
  "list"=>["l{2}"]
56
55
  }
57
-
56
+
58
57
  url = url_for(%w(ball foot move))
59
- assert_redacted @subject.remove_from_query(url, sample_config), %w(ball)
58
+ assert_redacted @subject.remove_from_query(url, sample_config), %w(ball)
59
+ end
60
+
61
+ def test_invalid
62
+ sample_config = {
63
+ "matcher"=>"test_invalid",
64
+ "list"=>["key"]
65
+ }
66
+
67
+ url = url_for(%w(key Str kEy KEY))
68
+ assert_redacted @subject.remove_from_query(url, sample_config), []
69
+ end
70
+
71
+ def test_without_scheme
72
+ sample_config = {
73
+ "matcher"=>"contains",
74
+ "list"=>["stan"]
75
+ }
76
+
77
+ url = 'example.com?instantiate=true'
78
+ assert_redacted @subject.remove_from_query(url, sample_config), %w(instantiate)
60
79
  end
61
-
80
+
62
81
  private
63
-
82
+
64
83
  def url_for(keys)
65
84
  url = URI('http://example.com')
66
85
  url.query = URI.encode_www_form(keys.map { |k| [k, rand(1..100)]})
67
86
  url.to_s
68
87
  end
69
-
88
+
70
89
  def assert_redacted(str, keys)
71
90
  url = URI(str)
72
91
  params = CGI.parse(url.query)
73
-
92
+
74
93
  assert_equal keys, params.select { |_, v| v == %w(<redacted>) }.keys, 'to be redacted'
75
94
  end
76
95
  end