hoss-agent 1.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE/Bug_report.md +40 -0
  3. data/.github/ISSUE_TEMPLATE/Feature_request.md +17 -0
  4. data/.github/PULL_REQUEST_TEMPLATE.md +60 -0
  5. data/.gitignore +27 -0
  6. data/.rspec +2 -0
  7. data/Dockerfile +43 -0
  8. data/Gemfile +105 -0
  9. data/LICENSE +201 -0
  10. data/hoss-agent.gemspec +42 -0
  11. data/lib/hoss-agent.rb +210 -0
  12. data/lib/hoss.rb +21 -0
  13. data/lib/hoss/agent.rb +235 -0
  14. data/lib/hoss/central_config.rb +184 -0
  15. data/lib/hoss/central_config/cache_control.rb +51 -0
  16. data/lib/hoss/child_durations.rb +64 -0
  17. data/lib/hoss/config.rb +315 -0
  18. data/lib/hoss/config/bytes.rb +42 -0
  19. data/lib/hoss/config/duration.rb +40 -0
  20. data/lib/hoss/config/options.rb +154 -0
  21. data/lib/hoss/config/regexp_list.rb +30 -0
  22. data/lib/hoss/config/wildcard_pattern_list.rb +54 -0
  23. data/lib/hoss/context.rb +64 -0
  24. data/lib/hoss/context/request.rb +28 -0
  25. data/lib/hoss/context/request/socket.rb +36 -0
  26. data/lib/hoss/context/request/url.rb +59 -0
  27. data/lib/hoss/context/response.rb +47 -0
  28. data/lib/hoss/context/user.rb +59 -0
  29. data/lib/hoss/context_builder.rb +112 -0
  30. data/lib/hoss/deprecations.rb +39 -0
  31. data/lib/hoss/error.rb +49 -0
  32. data/lib/hoss/error/exception.rb +70 -0
  33. data/lib/hoss/error/log.rb +41 -0
  34. data/lib/hoss/error_builder.rb +90 -0
  35. data/lib/hoss/event.rb +131 -0
  36. data/lib/hoss/instrumenter.rb +107 -0
  37. data/lib/hoss/internal_error.rb +23 -0
  38. data/lib/hoss/logging.rb +70 -0
  39. data/lib/hoss/metadata.rb +36 -0
  40. data/lib/hoss/metadata/process_info.rb +35 -0
  41. data/lib/hoss/metadata/service_info.rb +76 -0
  42. data/lib/hoss/metadata/system_info.rb +47 -0
  43. data/lib/hoss/metadata/system_info/container_info.rb +136 -0
  44. data/lib/hoss/naively_hashable.rb +38 -0
  45. data/lib/hoss/rails.rb +68 -0
  46. data/lib/hoss/railtie.rb +42 -0
  47. data/lib/hoss/report.rb +9 -0
  48. data/lib/hoss/sinatra.rb +53 -0
  49. data/lib/hoss/spies.rb +104 -0
  50. data/lib/hoss/spies/faraday.rb +117 -0
  51. data/lib/hoss/spies/http.rb +93 -0
  52. data/lib/hoss/spies/net_http.rb +113 -0
  53. data/lib/hoss/stacktrace.rb +33 -0
  54. data/lib/hoss/stacktrace/frame.rb +66 -0
  55. data/lib/hoss/stacktrace_builder.rb +124 -0
  56. data/lib/hoss/transport/base.rb +191 -0
  57. data/lib/hoss/transport/connection.rb +55 -0
  58. data/lib/hoss/transport/connection/http.rb +139 -0
  59. data/lib/hoss/transport/connection/proxy_pipe.rb +94 -0
  60. data/lib/hoss/transport/filters.rb +60 -0
  61. data/lib/hoss/transport/filters/hash_sanitizer.rb +77 -0
  62. data/lib/hoss/transport/filters/secrets_filter.rb +48 -0
  63. data/lib/hoss/transport/headers.rb +74 -0
  64. data/lib/hoss/transport/serializers.rb +113 -0
  65. data/lib/hoss/transport/serializers/context_serializer.rb +112 -0
  66. data/lib/hoss/transport/serializers/error_serializer.rb +92 -0
  67. data/lib/hoss/transport/serializers/event_serializer.rb +73 -0
  68. data/lib/hoss/transport/serializers/metadata_serializer.rb +92 -0
  69. data/lib/hoss/transport/serializers/report_serializer.rb +33 -0
  70. data/lib/hoss/transport/user_agent.rb +48 -0
  71. data/lib/hoss/transport/worker.rb +330 -0
  72. data/lib/hoss/util.rb +54 -0
  73. data/lib/hoss/util/inflector.rb +110 -0
  74. data/lib/hoss/util/lru_cache.rb +65 -0
  75. data/lib/hoss/util/throttle.rb +52 -0
  76. data/lib/hoss/version.rb +22 -0
  77. metadata +147 -0
@@ -0,0 +1,33 @@
1
+ # Licensed to Elasticsearch B.V. under one or more contributor
2
+ # license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright
4
+ # ownership. Elasticsearch B.V. licenses this file to you under
5
+ # the Apache License, Version 2.0 (the "License"); you may
6
+ # not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # frozen_string_literal: true
19
+
20
+ module Hoss
21
+ module Transport
22
+ module Serializers
23
+ # @api private
24
+ class ReportSerializer < Serializer
25
+ def build(report)
26
+ {
27
+ events: report.events
28
+ }
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,48 @@
1
+ # Licensed to Elasticsearch B.V. under one or more contributor
2
+ # license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright
4
+ # ownership. Elasticsearch B.V. licenses this file to you under
5
+ # the Apache License, Version 2.0 (the "License"); you may
6
+ # not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # frozen_string_literal: true
19
+
20
+ module Hoss
21
+ module Transport
22
+ # @api private
23
+ class UserAgent
24
+ def initialize(config)
25
+ @built = build(config)
26
+ end
27
+
28
+ def to_s
29
+ @built
30
+ end
31
+
32
+ private
33
+
34
+ def build(config)
35
+ metadata = Metadata.new(config)
36
+
37
+ [
38
+ "hoss-ruby/#{VERSION}",
39
+ HTTP::Request::USER_AGENT,
40
+ [
41
+ metadata.service.runtime.name,
42
+ metadata.service.runtime.version
43
+ ].join('/')
44
+ ].join(' ')
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,330 @@
1
+ # Licensed to Elasticsearch B.V. under one or more contributor
2
+ # license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright
4
+ # ownership. Elasticsearch B.V. licenses this file to you under
5
+ # the Apache License, Version 2.0 (the "License"); you may
6
+ # not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # frozen_string_literal: true
19
+
20
+ require 'hoss/report'
21
+ require 'zlib'
22
+ require 'cgi'
23
+
24
+ module Hoss
25
+ module Transport
26
+ # @api private
27
+ class Worker
28
+ include Logging
29
+
30
+ class << self
31
+ def adapter
32
+ @adapter ||= Connection
33
+ end
34
+
35
+ attr_writer :adapter
36
+ end
37
+
38
+ # @api private
39
+ class StopMessage; end
40
+
41
+ # @api private
42
+ class FlushMessage; end
43
+
44
+ def initialize(
45
+ config,
46
+ queue,
47
+ serializers:,
48
+ filters:
49
+ )
50
+ @config = config
51
+ @queue = queue
52
+
53
+ @serializers = serializers
54
+ @filters = filters
55
+
56
+ @connection = self.class.adapter.new(config)
57
+ end
58
+
59
+ attr_reader :config, :queue, :filters, :name, :connection, :serializers
60
+ def work_forever
61
+
62
+ done = false
63
+ while (!done && msg = queue.pop)
64
+ begin
65
+ debug 'working message', msg
66
+
67
+ # wait if we don't have a config
68
+ while config.agentConfig.nil?
69
+ sleep 0.1
70
+ end
71
+
72
+ case msg
73
+ when StopMessage
74
+ debug 'Stopping worker -- %s', self
75
+ done = true
76
+ break
77
+ else
78
+ batch = []
79
+ current_batch_size = 0
80
+
81
+ # Use this as the first message in the batch
82
+ event = msg.filtered ? msg : filter_resource(msg)
83
+ unless event.nil?
84
+ event_size = resource_size(event)
85
+ if current_batch_size + event_size <= @config.batch_size
86
+ unless host_blacklisted(event)
87
+ current_batch_size += event_size
88
+ if event.retries < @config.max_event_retries
89
+ batch.push(event)
90
+ else
91
+ debug "max retries hit for event"
92
+ end
93
+ end
94
+ else
95
+ debug "Event is too large, body needs to be truncated"
96
+ end
97
+ end
98
+
99
+ # Do inner loop reading queue to build report
100
+ requeue_messages = []
101
+ while current_batch_size < @config.batch_size && !queue.empty?
102
+ next_msg = queue.pop
103
+ case next_msg
104
+ when StopMessage
105
+ debug 'Stopping worker -- %s', self
106
+ done = true
107
+ break
108
+ else
109
+ event = next_msg.filtered ? next_msg : filter_resource(next_msg)
110
+ unless event.nil?
111
+ event_size = resource_size(event)
112
+ if current_batch_size + event_size <= @config.batch_size
113
+ unless host_blacklisted(event)
114
+ current_batch_size += event_size
115
+ if event.retries < @config.max_event_retries
116
+ batch.push(event)
117
+ else
118
+ debug "max retries hit for event"
119
+ end
120
+ end
121
+ else
122
+ debug "Event too large for this batch, requeue"
123
+ requeue_messages.push(event)
124
+ end
125
+ end
126
+ end
127
+ end
128
+
129
+ if batch.length == 0
130
+ debug "batch is empty, breaking"
131
+ break
132
+ end
133
+
134
+ debug "Requeue #{requeue_messages.length} messages" if requeue_messages.length > 0
135
+ requeue_messages.each {|msg| queue.push(msg, false) }
136
+
137
+ report = Report.new
138
+ report.events = batch.map {|event| serializers.serialize(event) }
139
+
140
+ debug "Finished building report"
141
+ data = serializers.serialize(report)
142
+ json = JSON.fast_generate(data)
143
+ begin
144
+ debug json
145
+ rescue Exception => e
146
+ debug 'unable to print body'
147
+ puts json if config.debug
148
+ end
149
+ begin
150
+ if config.disable_reporting
151
+ debug "Reprting disabled, skipping"
152
+ else
153
+ connection.write(json)
154
+ end
155
+ rescue Exception => e
156
+ error format('Failed send report: %s %s', e.inspect, e.backtrace)
157
+ batch.each do |m|
158
+ m.retries += 1
159
+ queue.push(m, false)
160
+ end
161
+ sleep 1
162
+ end
163
+ end
164
+ rescue Exception => e
165
+ debug "error in worker #{e.inspect}"
166
+ end
167
+ end
168
+ rescue Exception => e
169
+ warn 'Worker died with exception: %s', e.inspect
170
+ debug e.backtrace.join("\n")
171
+ end
172
+
173
+ private
174
+
175
+ def resource_size(resource)
176
+ size = 20
177
+ size += JSON.fast_generate(resource.request.headers).length if resource.request && resource.request.headers
178
+ size += JSON.fast_generate(resource.response.headers).length if resource.response && resource.response.headers
179
+ size += resource.request.body.length if resource.request && resource.request.body
180
+ size += resource.response.body.length if resource.response && resource.response.body
181
+ size
182
+ end
183
+
184
+ def decompress_body(body, encoding)
185
+ return body unless body
186
+ case (encoding || '').downcase
187
+ when "gzip"
188
+ Zlib::GzipReader.new(StringIO.new(body)).read.to_s
189
+ when "deflate"
190
+ Zlib::Inflate.new.inflate(body)
191
+ else
192
+ body
193
+ end
194
+ rescue Exception => e
195
+ debug "unable to decompress body #{e.inspect}"
196
+ return nil
197
+ end
198
+
199
+ def get_header(headers, name)
200
+ headers[headers.keys.select{|h| h.casecmp(name) == 0}[0]]
201
+ end
202
+
203
+ def filter_resource(resource)
204
+ # Make sure the body is not compressed
205
+ resource.request.body = decompress_body(resource.request.body, get_header(resource.request.headers, "content-encoding")) if resource.request
206
+ resource.response.body = decompress_body(resource.response.body, get_header(resource.response.headers,"content-encoding")) if resource.response
207
+
208
+ # Start applying filters
209
+ config = get_configuration_for_host(resource.request.headers['host'])
210
+ debug "Using config for event #{config.to_s}"
211
+
212
+ # Filter the headers
213
+ filter_headers(resource.request.headers, config['sanitizedHeaders']) if resource.request
214
+ filter_headers(resource.response.headers, config['sanitizedHeaders']) if resource.response
215
+
216
+ # Filter the body
217
+ if config['bodyCapture'] == 'Off' || (config['bodyCapture'] == 'OnError' && !contains_error(resource))
218
+ resource.request.body = nil if resource.request
219
+ resource.response.body = nil if resource.response
220
+ else
221
+ resource.request.body = filter_body(resource.request.body, config['sanitizedBodyFields']) if resource.request
222
+ resource.response.body = filter_body(resource.response.body, config['sanitizedBodyFields']) if resource.response
223
+ end
224
+
225
+ # Filter the query params
226
+ resource.request.url = filter_url(resource.request.url, config['sanitizedQueryParams'])
227
+
228
+ # Record this event as filtered in case of retry
229
+ resource.filtered = true
230
+
231
+ resource
232
+ end
233
+
234
+ def contains_error(resource)
235
+ if !resource.response.nil? && resource.response.status_code >= 400
236
+ return true
237
+ end
238
+ false
239
+ end
240
+
241
+ def filter_url(url, config)
242
+ uri = URI(url)
243
+ if uri.query
244
+ query = CGI::parse(uri.query)
245
+ query.keys.each{|k| query[k] = 'xxx' if contains_string(config, k)}
246
+ uri.query = URI.encode_www_form(query)
247
+ return uri.to_s
248
+ end
249
+ url
250
+ rescue Exception => e
251
+ error "Error filtering url", e
252
+ url
253
+ end
254
+
255
+ def filter_headers(headers, names)
256
+ headers.each { |h| headers[h[0]] = 'xxx' if contains_string(names, h[0]) }
257
+ end
258
+
259
+ def filter_body(body, fields)
260
+ return nil if body.nil?
261
+ json = JSON.parse(body)
262
+ JSON.fast_generate(mask_object_fields(json, fields))
263
+ rescue Exception => e
264
+ body
265
+ end
266
+
267
+ def mask_object_fields(obj, fields)
268
+ field_names = fields.map {|f| f['value'] }
269
+ case obj
270
+ when Hash
271
+ obj.keys.each do |k|
272
+ if contains_string(field_names, k)
273
+ obj[k] = 'xxx'
274
+ else
275
+ obj[k] = mask_object_fields(obj[k], fields)
276
+ end
277
+ end
278
+ when Array
279
+ obj.each_index do |i|
280
+ obj[i] = mask_object_fields(obj[i], fields) if (obj[i].is_a?(Hash) || obj[i].is_a?(Array))
281
+ end
282
+ end
283
+ obj
284
+ end
285
+
286
+ # Get the configuration for a host, either use a defined api or the default account config
287
+ def get_configuration_for_host(host)
288
+ api_config = api_configurations.select{ |api| contains_string(api['hosts'], host)}[0]
289
+ return api_config['configuration'] if api_config
290
+ account_api_configuration
291
+ end
292
+
293
+ # Parse out the host blacklist
294
+ def get_account_host_blacklist
295
+ account_api_configuration['hostBlacklist']
296
+ end
297
+
298
+ # Do a case insensitive search for a string in array
299
+ def contains_string(arr, a)
300
+ !arr.select { |b| b.casecmp(a) == 0 }.empty?
301
+ end
302
+
303
+ # Return 'accountApiConfiguration' or empty object
304
+ def account_api_configuration
305
+ return {} if config.agentConfig.nil? or config.agentConfig['accountApiConfiguration'].nil?
306
+ config.agentConfig['accountApiConfiguration']
307
+ end
308
+
309
+ # Return the 'apis' or empty object
310
+ def api_configurations
311
+ return {} if config.agentConfig.nil? or config.agentConfig['apis'].nil?
312
+ config.agentConfig['apis']
313
+ end
314
+
315
+ def host_blacklisted(event)
316
+ blacklist = get_account_host_blacklist
317
+ return false if blacklist.nil?
318
+ is_blacklisted = false
319
+ unless event.request.nil? || event.request.url.nil?
320
+ url = URI(event.request.url)
321
+ blacklist.each do |host|
322
+ is_blacklisted = true if url.host == host
323
+ is_blacklisted = true if url.host.end_with? '.'+host
324
+ end
325
+ end
326
+ is_blacklisted
327
+ end
328
+ end
329
+ end
330
+ end
@@ -0,0 +1,54 @@
1
+ # Licensed to Elasticsearch B.V. under one or more contributor
2
+ # license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright
4
+ # ownership. Elasticsearch B.V. licenses this file to you under
5
+ # the Apache License, Version 2.0 (the "License"); you may
6
+ # not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ # frozen_string_literal: true
19
+
20
+ module Hoss
21
+ # @api private
22
+ module Util
23
+ def self.micros(target = Time.now)
24
+ utc = target.utc
25
+ utc.to_i * 1_000_000 + utc.usec
26
+ end
27
+
28
+ def self.monotonic_micros
29
+ Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond)
30
+ end
31
+
32
+ def self.git_sha
33
+ sha = `git rev-parse --verify HEAD 2>&1`.chomp
34
+ $?&.success? ? sha : nil
35
+ end
36
+
37
+ def self.hex_to_bits(str)
38
+ str.hex.to_s(2).rjust(str.size * 4, '0')
39
+ end
40
+
41
+ def self.reverse_merge!(first, *others)
42
+ others.reduce(first) do |curr, other|
43
+ curr.merge!(other) { |_, _, new| new }
44
+ end
45
+ end
46
+
47
+ def self.truncate(value, max_length: 1024)
48
+ return unless value
49
+ return value if value.length <= max_length
50
+
51
+ value[0...(max_length - 1)] + '…'
52
+ end
53
+ end
54
+ end