libhoney 1.18.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +7 -4
- data/.github/CODEOWNERS +1 -1
- data/.github/ISSUE_TEMPLATE/bug_report.md +27 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +25 -0
- data/.github/ISSUE_TEMPLATE/question-discussion.md +14 -0
- data/.github/ISSUE_TEMPLATE/security-vulnerability-report.md +22 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +20 -0
- data/.github/dependabot.yml +15 -0
- data/.github/workflows/add-to-project.yml +14 -0
- data/.github/workflows/apply-labels.yml +10 -0
- data/.github/workflows/stale.yml +26 -0
- data/.gitignore +1 -0
- data/.rubocop.yml +27 -4
- data/.rubocop_todo.yml +24 -18
- data/CHANGELOG.md +43 -2
- data/CODE_OF_CONDUCT.md +5 -0
- data/CONTRIBUTING.md +3 -0
- data/CONTRIBUTORS +1 -0
- data/OSSMETADATA +1 -0
- data/README.md +6 -14
- data/RELEASING.md +12 -0
- data/SECURITY.md +3 -0
- data/SUPPORT.md +3 -0
- data/example/factorial.rb +1 -1
- data/lib/libhoney/builder.rb +1 -2
- data/lib/libhoney/client.rb +70 -56
- data/lib/libhoney/experimental_transmission.rb +107 -0
- data/lib/libhoney/log_client.rb +1 -1
- data/lib/libhoney/mock_transmission.rb +1 -1
- data/lib/libhoney/null_transmission.rb +2 -0
- data/lib/libhoney/queueing/LICENSE.txt +23 -0
- data/lib/libhoney/queueing/sized_queue_with_timeout.rb +176 -0
- data/lib/libhoney/queueing.rb +1 -0
- data/lib/libhoney/transmission.rb +109 -48
- data/lib/libhoney/version.rb +1 -1
- data/libhoney.gemspec +7 -5
- metadata +63 -16
@@ -36,9 +36,9 @@ module Libhoney
|
|
36
36
|
@send_queue = Queue.new
|
37
37
|
@threads = []
|
38
38
|
@lock = Mutex.new
|
39
|
-
# use a SizedQueue so the producer will block on adding to the batch_queue when @block_on_send is true
|
40
|
-
@batch_queue = SizedQueue.new(@pending_work_capacity)
|
41
39
|
@batch_thread = nil
|
40
|
+
|
41
|
+
setup_batch_queue
|
42
42
|
end
|
43
43
|
|
44
44
|
def add(event)
|
@@ -53,26 +53,6 @@ module Libhoney
|
|
53
53
|
ensure_threads_running
|
54
54
|
end
|
55
55
|
|
56
|
-
def event_valid(event)
|
57
|
-
invalid = []
|
58
|
-
invalid.push('api host') if event.api_host.nil? || event.api_host.empty?
|
59
|
-
invalid.push('write key') if event.writekey.nil? || event.writekey.empty?
|
60
|
-
invalid.push('dataset') if event.dataset.nil? || event.dataset.empty?
|
61
|
-
|
62
|
-
unless invalid.empty?
|
63
|
-
e = StandardError.new("#{self.class.name}: nil or empty required fields (#{invalid.join(', ')})"\
|
64
|
-
'. Will not attempt to send.')
|
65
|
-
Response.new(error: e).tap do |error_response|
|
66
|
-
error_response.metadata = event.metadata
|
67
|
-
enqueue_response(error_response)
|
68
|
-
end
|
69
|
-
|
70
|
-
return false
|
71
|
-
end
|
72
|
-
|
73
|
-
true
|
74
|
-
end
|
75
|
-
|
76
56
|
def send_loop
|
77
57
|
http_clients = build_http_clients
|
78
58
|
|
@@ -133,23 +113,28 @@ module Libhoney
|
|
133
113
|
end
|
134
114
|
|
135
115
|
def close(drain)
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
116
|
+
@lock.synchronize do
|
117
|
+
# if drain is false, clear the remaining unprocessed events from the queue
|
118
|
+
if drain
|
119
|
+
warn "#{self.class.name} - close: draining events" if %w[debug trace].include?(ENV['LOG_LEVEL'])
|
120
|
+
else
|
121
|
+
warn "#{self.class.name} - close: deleting unsent events" if %w[debug trace].include?(ENV['LOG_LEVEL'])
|
122
|
+
@batch_queue.clear
|
123
|
+
@send_queue.clear
|
124
|
+
end
|
141
125
|
|
142
|
-
|
143
|
-
|
126
|
+
@batch_queue.enq(nil)
|
127
|
+
@batch_thread&.join(1.0) # limit the amount of time we'll wait for the thread to end
|
144
128
|
|
145
|
-
|
146
|
-
|
129
|
+
# send @threads.length number of nils so each thread will fall out of send_loop
|
130
|
+
@threads.length.times { @send_queue << nil }
|
147
131
|
|
148
|
-
|
149
|
-
|
132
|
+
@threads.each(&:join)
|
133
|
+
@threads = []
|
134
|
+
end
|
150
135
|
|
151
136
|
enqueue_response(nil)
|
152
|
-
|
137
|
+
warn "#{self.class.name} - close: close complete" if %w[debug trace].include?(ENV['LOG_LEVEL'])
|
153
138
|
0
|
154
139
|
end
|
155
140
|
|
@@ -161,25 +146,79 @@ module Libhoney
|
|
161
146
|
|
162
147
|
loop do
|
163
148
|
begin
|
149
|
+
# a timeout expiration waiting for an event
|
150
|
+
# 1. interrupts only when thread is in a blocking state (waiting for pop)
|
151
|
+
# 2. exception skips the break and is rescued
|
152
|
+
# 3. triggers the ensure to flush the current batch
|
153
|
+
# 3. begins the loop again with an updated next_send_time
|
164
154
|
Thread.handle_interrupt(Timeout::Error => :on_blocking) do
|
155
|
+
# an event on the batch_queue
|
156
|
+
# 1. pops out and is truthy
|
157
|
+
# 2. gets included in the current batch
|
158
|
+
# 3. while waits for another event
|
165
159
|
while (event = Timeout.timeout(@send_frequency) { @batch_queue.pop })
|
166
160
|
key = [event.api_host, event.writekey, event.dataset]
|
167
161
|
batched_events[key] << event
|
168
162
|
end
|
169
163
|
end
|
170
164
|
|
165
|
+
# a nil on the batch_queue
|
166
|
+
# 1. pops out and is falsy
|
167
|
+
# 2. ends the event-popping while do..end
|
168
|
+
# 3. breaks the loop
|
169
|
+
# 4. flushes the current batch
|
170
|
+
# 5. ends the batch_loop
|
171
171
|
break
|
172
|
-
rescue
|
172
|
+
rescue Timeout::Error
|
173
|
+
# Timeout::Error happens when there is nothing to pop from the batch_queue.
|
174
|
+
# We rescue it here to avoid spamming the logs with "execution expired" errors.
|
175
|
+
rescue Exception => e
|
176
|
+
warn "#{self.class.name}: 💥 " + e.message if %w[debug trace].include?(ENV['LOG_LEVEL'])
|
177
|
+
warn e.backtrace.join("\n").to_s if ['trace'].include?(ENV['LOG_LEVEL'])
|
178
|
+
|
179
|
+
# regardless of the exception, figure out whether enough time has passed to
|
180
|
+
# send the current batched events, if so, send them and figure out the next send time
|
181
|
+
# before going back to the top of the loop
|
173
182
|
ensure
|
174
183
|
next_send_time = flush_batched_events(batched_events) if Time.now > next_send_time
|
175
184
|
end
|
176
185
|
end
|
177
186
|
|
187
|
+
# don't need to capture the next_send_time here because the batch_loop is exiting
|
188
|
+
# for some reason (probably transmission.close)
|
178
189
|
flush_batched_events(batched_events)
|
179
190
|
end
|
180
191
|
|
181
192
|
private
|
182
193
|
|
194
|
+
def setup_batch_queue
|
195
|
+
# use a SizedQueue so the producer will block on adding to the batch_queue when @block_on_send is true
|
196
|
+
@batch_queue = SizedQueue.new(@pending_work_capacity)
|
197
|
+
end
|
198
|
+
|
199
|
+
REQUIRED_EVENT_FIELDS = %i[api_host writekey dataset].freeze
|
200
|
+
|
201
|
+
def event_valid(event)
|
202
|
+
missing_required_fields = REQUIRED_EVENT_FIELDS.select do |required_field|
|
203
|
+
event.public_send(required_field).nil? || event.public_send(required_field).empty?
|
204
|
+
end
|
205
|
+
|
206
|
+
if missing_required_fields.empty?
|
207
|
+
true
|
208
|
+
else
|
209
|
+
enqueue_response(
|
210
|
+
Response.new(
|
211
|
+
metadata: event.metadata,
|
212
|
+
error: StandardError.new(
|
213
|
+
"#{self.class.name}: nil or empty required fields (#{missing_required_fields.join(', ')})"\
|
214
|
+
'. Will not attempt to send.'
|
215
|
+
)
|
216
|
+
)
|
217
|
+
)
|
218
|
+
false
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
183
222
|
##
|
184
223
|
# Enqueues a response to the responses queue suppressing ThreadError when
|
185
224
|
# there is no space left on the queue and we are not blocking on response
|
@@ -190,15 +229,36 @@ module Libhoney
|
|
190
229
|
end
|
191
230
|
|
192
231
|
def process_response(http_response, before, batch)
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
232
|
+
if http_response.status == 200
|
233
|
+
index = 0
|
234
|
+
JSON.parse(http_response.body).each do |event|
|
235
|
+
index += 1 while batch[index].nil? && index < batch.size
|
236
|
+
break unless (batched_event = batch[index])
|
237
|
+
|
238
|
+
enqueue_response(
|
239
|
+
Response.new(
|
240
|
+
status_code: event['status'],
|
241
|
+
duration: (Time.now - before),
|
242
|
+
metadata: batched_event.metadata
|
243
|
+
)
|
244
|
+
)
|
245
|
+
end
|
246
|
+
else
|
247
|
+
error = JSON.parse(http_response.body)['error']
|
248
|
+
if %w[debug trace].include?(ENV['LOG_LEVEL'])
|
249
|
+
warn "#{self.class.name}: error sending data to Honeycomb - #{http_response.status} #{error}"
|
250
|
+
end
|
251
|
+
batch.each do |batched_event|
|
252
|
+
next unless batched_event # skip nils enqueued from serialization errors
|
253
|
+
|
254
|
+
enqueue_response(
|
255
|
+
Response.new(
|
256
|
+
status_code: http_response.status, # single error from API applied to all events sent in batch
|
257
|
+
duration: (Time.now - before),
|
258
|
+
metadata: batched_event.metadata,
|
259
|
+
error: RuntimeError.new(error)
|
260
|
+
)
|
261
|
+
)
|
202
262
|
end
|
203
263
|
end
|
204
264
|
end
|
@@ -234,14 +294,15 @@ module Libhoney
|
|
234
294
|
end
|
235
295
|
|
236
296
|
def build_user_agent(user_agent_addition)
|
237
|
-
|
238
|
-
|
239
|
-
|
297
|
+
"libhoney-rb/#{VERSION}"
|
298
|
+
.concat(" #{user_agent_addition}")
|
299
|
+
.strip # remove trailing spaces if addition was empty
|
300
|
+
.concat(" Ruby/#{RUBY_VERSION} (#{RUBY_PLATFORM})")
|
240
301
|
end
|
241
302
|
|
242
303
|
def ensure_threads_running
|
243
304
|
@lock.synchronize do
|
244
|
-
@batch_thread = Thread.new { batch_loop } unless @batch_thread
|
305
|
+
@batch_thread = Thread.new { batch_loop } unless @batch_thread&.alive?
|
245
306
|
@threads.select!(&:alive?)
|
246
307
|
@threads << Thread.new { send_loop } while @threads.length < @max_concurrent_batches
|
247
308
|
end
|
data/lib/libhoney/version.rb
CHANGED
data/libhoney.gemspec
CHANGED
@@ -20,20 +20,22 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
21
|
spec.require_paths = ['lib']
|
22
22
|
|
23
|
-
spec.required_ruby_version = '>= 2.
|
23
|
+
spec.required_ruby_version = '>= 2.4.0'
|
24
24
|
|
25
25
|
spec.add_development_dependency 'bump', '~> 0.5'
|
26
26
|
spec.add_development_dependency 'bundler'
|
27
|
+
spec.add_development_dependency 'lockstep'
|
27
28
|
spec.add_development_dependency 'minitest', '~> 5.0'
|
28
|
-
spec.add_development_dependency '
|
29
|
-
spec.add_development_dependency '
|
29
|
+
spec.add_development_dependency 'minitest-reporters'
|
30
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
31
|
+
spec.add_development_dependency 'rubocop', '1.12.1'
|
30
32
|
spec.add_development_dependency 'sinatra'
|
31
33
|
spec.add_development_dependency 'sinatra-contrib'
|
32
|
-
spec.add_development_dependency 'spy', '1.0
|
34
|
+
spec.add_development_dependency 'spy', '~> 1.0'
|
33
35
|
spec.add_development_dependency 'webmock', '~> 3.4'
|
34
36
|
spec.add_development_dependency 'yard'
|
35
37
|
spec.add_development_dependency 'yardstick', '~> 0.9'
|
36
38
|
spec.add_dependency 'addressable', '~> 2.0'
|
37
39
|
spec.add_dependency 'excon'
|
38
|
-
spec.add_dependency 'http', '>= 2.0', '<
|
40
|
+
spec.add_dependency 'http', '>= 2.0', '< 6.0'
|
39
41
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: libhoney
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- The Honeycomb.io Team
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-10-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bump
|
@@ -38,6 +38,20 @@ dependencies:
|
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: lockstep
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: minitest
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -52,34 +66,48 @@ dependencies:
|
|
52
66
|
- - "~>"
|
53
67
|
- !ruby/object:Gem::Version
|
54
68
|
version: '5.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: minitest-reporters
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
55
83
|
- !ruby/object:Gem::Dependency
|
56
84
|
name: rake
|
57
85
|
requirement: !ruby/object:Gem::Requirement
|
58
86
|
requirements:
|
59
87
|
- - "~>"
|
60
88
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
89
|
+
version: '13.0'
|
62
90
|
type: :development
|
63
91
|
prerelease: false
|
64
92
|
version_requirements: !ruby/object:Gem::Requirement
|
65
93
|
requirements:
|
66
94
|
- - "~>"
|
67
95
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
96
|
+
version: '13.0'
|
69
97
|
- !ruby/object:Gem::Dependency
|
70
98
|
name: rubocop
|
71
99
|
requirement: !ruby/object:Gem::Requirement
|
72
100
|
requirements:
|
73
|
-
- -
|
101
|
+
- - '='
|
74
102
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
103
|
+
version: 1.12.1
|
76
104
|
type: :development
|
77
105
|
prerelease: false
|
78
106
|
version_requirements: !ruby/object:Gem::Requirement
|
79
107
|
requirements:
|
80
|
-
- -
|
108
|
+
- - '='
|
81
109
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
110
|
+
version: 1.12.1
|
83
111
|
- !ruby/object:Gem::Dependency
|
84
112
|
name: sinatra
|
85
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -112,16 +140,16 @@ dependencies:
|
|
112
140
|
name: spy
|
113
141
|
requirement: !ruby/object:Gem::Requirement
|
114
142
|
requirements:
|
115
|
-
- -
|
143
|
+
- - "~>"
|
116
144
|
- !ruby/object:Gem::Version
|
117
|
-
version: 1.0
|
145
|
+
version: '1.0'
|
118
146
|
type: :development
|
119
147
|
prerelease: false
|
120
148
|
version_requirements: !ruby/object:Gem::Requirement
|
121
149
|
requirements:
|
122
|
-
- -
|
150
|
+
- - "~>"
|
123
151
|
- !ruby/object:Gem::Version
|
124
|
-
version: 1.0
|
152
|
+
version: '1.0'
|
125
153
|
- !ruby/object:Gem::Dependency
|
126
154
|
name: webmock
|
127
155
|
requirement: !ruby/object:Gem::Requirement
|
@@ -201,7 +229,7 @@ dependencies:
|
|
201
229
|
version: '2.0'
|
202
230
|
- - "<"
|
203
231
|
- !ruby/object:Gem::Version
|
204
|
-
version: '
|
232
|
+
version: '6.0'
|
205
233
|
type: :runtime
|
206
234
|
prerelease: false
|
207
235
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -211,7 +239,7 @@ dependencies:
|
|
211
239
|
version: '2.0'
|
212
240
|
- - "<"
|
213
241
|
- !ruby/object:Gem::Version
|
214
|
-
version: '
|
242
|
+
version: '6.0'
|
215
243
|
description: Ruby gem for sending data to Honeycomb
|
216
244
|
email: support@honeycomb.io
|
217
245
|
executables: []
|
@@ -222,27 +250,46 @@ files:
|
|
222
250
|
- ".circleci/setup-rubygems.sh"
|
223
251
|
- ".editorconfig"
|
224
252
|
- ".github/CODEOWNERS"
|
253
|
+
- ".github/ISSUE_TEMPLATE/bug_report.md"
|
254
|
+
- ".github/ISSUE_TEMPLATE/feature_request.md"
|
255
|
+
- ".github/ISSUE_TEMPLATE/question-discussion.md"
|
256
|
+
- ".github/ISSUE_TEMPLATE/security-vulnerability-report.md"
|
257
|
+
- ".github/PULL_REQUEST_TEMPLATE.md"
|
258
|
+
- ".github/dependabot.yml"
|
259
|
+
- ".github/workflows/add-to-project.yml"
|
260
|
+
- ".github/workflows/apply-labels.yml"
|
261
|
+
- ".github/workflows/stale.yml"
|
225
262
|
- ".gitignore"
|
226
263
|
- ".rubocop.yml"
|
227
264
|
- ".rubocop_todo.yml"
|
228
265
|
- CHANGELOG.md
|
266
|
+
- CODE_OF_CONDUCT.md
|
267
|
+
- CONTRIBUTING.md
|
229
268
|
- CONTRIBUTORS
|
230
269
|
- Gemfile
|
231
270
|
- LICENSE
|
232
271
|
- NOTICE
|
272
|
+
- OSSMETADATA
|
233
273
|
- README.md
|
274
|
+
- RELEASING.md
|
234
275
|
- Rakefile
|
276
|
+
- SECURITY.md
|
277
|
+
- SUPPORT.md
|
235
278
|
- example/factorial.rb
|
236
279
|
- lib/libhoney.rb
|
237
280
|
- lib/libhoney/builder.rb
|
238
281
|
- lib/libhoney/cleaner.rb
|
239
282
|
- lib/libhoney/client.rb
|
240
283
|
- lib/libhoney/event.rb
|
284
|
+
- lib/libhoney/experimental_transmission.rb
|
241
285
|
- lib/libhoney/log_client.rb
|
242
286
|
- lib/libhoney/log_transmission.rb
|
243
287
|
- lib/libhoney/mock_transmission.rb
|
244
288
|
- lib/libhoney/null_client.rb
|
245
289
|
- lib/libhoney/null_transmission.rb
|
290
|
+
- lib/libhoney/queueing.rb
|
291
|
+
- lib/libhoney/queueing/LICENSE.txt
|
292
|
+
- lib/libhoney/queueing/sized_queue_with_timeout.rb
|
246
293
|
- lib/libhoney/response.rb
|
247
294
|
- lib/libhoney/test_client.rb
|
248
295
|
- lib/libhoney/transmission.rb
|
@@ -260,14 +307,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
260
307
|
requirements:
|
261
308
|
- - ">="
|
262
309
|
- !ruby/object:Gem::Version
|
263
|
-
version: 2.
|
310
|
+
version: 2.4.0
|
264
311
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
265
312
|
requirements:
|
266
313
|
- - ">="
|
267
314
|
- !ruby/object:Gem::Version
|
268
315
|
version: '0'
|
269
316
|
requirements: []
|
270
|
-
rubygems_version: 3.
|
317
|
+
rubygems_version: 3.2.22
|
271
318
|
signing_key:
|
272
319
|
specification_version: 4
|
273
320
|
summary: send data to Honeycomb
|