timber 2.1.10 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -2
- data/lib/timber/log_devices/http.rb +45 -41
- data/lib/timber/log_devices/http/{flushable_sized_queue.rb → flushable_dropping_sized_queue.rb} +16 -6
- data/lib/timber/log_devices/http/request_attempt.rb +20 -0
- data/lib/timber/version.rb +1 -1
- data/spec/timber/log_devices/http_spec.rb +18 -6
- metadata +4 -4
- data/lib/timber/log_devices/http/dropping_sized_queue.rb +0 -26
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d36593e9185855c949d8675ac749043317998c3c
|
4
|
+
data.tar.gz: 479e0355abf6e918aa1d0e0566e152c5fcfdaf97
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 70397717e17995dcbf20b2d667037307a3f19c9702325442ea896e2d0c7c424adb4eea1bb7bba4e7b70d511acf11235018b639e2b7aef72b800e11965f6d0f8c
|
7
|
+
data.tar.gz: b4f322a1167c530eb1b5e0a90fc836781619a93d0f7a2e05ce4bcbb342bb53ef2356195c8df8ea5055f75cbeac63c19fb7598b891d6a87c8210e7c510f3b8378
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,20 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
-
|
3
|
+
All notable changes to this project will be documented in this file.
|
4
4
|
|
5
|
-
|
5
|
+
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
|
6
|
+
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
|
7
|
+
|
8
|
+
## [Unreleased]
|
9
|
+
|
10
|
+
## [2.2.0] - 2017-09-13
|
11
|
+
### Changed
|
12
|
+
|
13
|
+
- The default HTTP log device queue type was switched to a
|
14
|
+
`Timber::LogDevices::HTTP::FlushableDroppingSizedQueue` instead of a `::SizedQueue`. In the
|
15
|
+
event of extremely high volume logging, and delivery cannot keep up, Timber will drop messages
|
16
|
+
instead of applying back pressure.
|
17
|
+
|
18
|
+
|
19
|
+
[Unreleased]: https://github.com/timberio/timber-ruby/compare/v2.2.0...HEAD
|
20
|
+
[2.2.0]: https://github.com/timberio/timber-ruby/compare/v2.1.10...v2.2.0
|
@@ -3,8 +3,8 @@ require "msgpack"
|
|
3
3
|
require "net/https"
|
4
4
|
|
5
5
|
require "timber/config"
|
6
|
-
require "timber/log_devices/http/
|
7
|
-
require "timber/log_devices/http/
|
6
|
+
require "timber/log_devices/http/flushable_dropping_sized_queue"
|
7
|
+
require "timber/log_devices/http/request_attempt"
|
8
8
|
require "timber/version"
|
9
9
|
|
10
10
|
module Timber
|
@@ -52,10 +52,11 @@ module Timber
|
|
52
52
|
# @option attributes [Symbol] :requests_per_conn (2500) The number of requests to send over a
|
53
53
|
# single persistent connection. After this number is met, the connection will be closed
|
54
54
|
# and a new one will be opened.
|
55
|
-
# @option attributes [Symbol] :request_queue (
|
56
|
-
# requests for delivery. By deafult this is a
|
57
|
-
#
|
58
|
-
# you'd prefer to
|
55
|
+
# @option attributes [Symbol] :request_queue (FlushableDroppingSizedQueue.new(25)) The request
|
56
|
+
# queue object that queues Net::HTTP requests for delivery. By deafult this is a
|
57
|
+
# `FlushableDroppingSizedQueue` of size `25`. Meaning once the queue fills up to 25
|
58
|
+
# requests new requests will be dropped. If you'd prefer to apply back pressure,
|
59
|
+
# ensuring you do not lose log data, pass a standard {SizedQueue}. See examples for
|
59
60
|
# an example.
|
60
61
|
# @option attributes [Symbol] :timber_url The Timber URL to delivery the log lines. The
|
61
62
|
# default is set via {TIMBER_URL}.
|
@@ -63,19 +64,18 @@ module Timber
|
|
63
64
|
# @example Basic usage
|
64
65
|
# Timber::Logger.new(Timber::LogDevices::HTTP.new("my_timber_api_key"))
|
65
66
|
#
|
66
|
-
# @example
|
67
|
-
# http_log_device = Timber::LogDevices::HTTP.new("my_timber_api_key",
|
68
|
-
# request_queue: Timber::LogDevices::HTTP::DroppingSizedQueue.new(3))
|
67
|
+
# @example Apply back pressure instead of dropping messages
|
68
|
+
# http_log_device = Timber::LogDevices::HTTP.new("my_timber_api_key", request_queue: SizedQueue.new(25))
|
69
69
|
# Timber::Logger.new(http_log_device)
|
70
70
|
def initialize(api_key, options = {})
|
71
71
|
@api_key = api_key || raise(ArgumentError.new("The api_key parameter cannot be blank"))
|
72
72
|
@timber_url = URI.parse(options[:timber_url] || ENV['TIMBER_URL'] || TIMBER_URL)
|
73
73
|
@batch_size = options[:batch_size] || 1_000
|
74
74
|
@flush_continuously = options[:flush_continuously] != false
|
75
|
-
@flush_interval = options[:flush_interval] ||
|
75
|
+
@flush_interval = options[:flush_interval] || 2 # 2 seconds
|
76
76
|
@requests_per_conn = options[:requests_per_conn] || 2_500
|
77
|
-
@msg_queue =
|
78
|
-
@request_queue = options[:request_queue] ||
|
77
|
+
@msg_queue = FlushableDroppingSizedQueue.new(@batch_size)
|
78
|
+
@request_queue = options[:request_queue] || FlushableDroppingSizedQueue.new(25)
|
79
79
|
@successive_error_count = 0
|
80
80
|
@requests_in_flight = 0
|
81
81
|
end
|
@@ -85,7 +85,7 @@ module Timber
|
|
85
85
|
# size is constricted by the Timber API. The actual application limit is a multiple
|
86
86
|
# of this. Hence the `@request_queue`.
|
87
87
|
def write(msg)
|
88
|
-
@msg_queue.
|
88
|
+
@msg_queue.enq(msg)
|
89
89
|
|
90
90
|
# Lazily start flush threads to ensure threads are alive after forking processes.
|
91
91
|
# If the threads are started during instantiation they will not be copied when
|
@@ -161,7 +161,8 @@ module Timber
|
|
161
161
|
req = build_request
|
162
162
|
if !req.nil?
|
163
163
|
Timber::Config.instance.debug { "New request placed on queue" }
|
164
|
-
|
164
|
+
request_attempt = RequestAttempt.new(req)
|
165
|
+
@request_queue.enq(request_attempt)
|
165
166
|
end
|
166
167
|
end
|
167
168
|
|
@@ -255,36 +256,39 @@ module Timber
|
|
255
256
|
while num_reqs < @requests_per_conn
|
256
257
|
Timber::Config.instance.debug { "Waiting on next request, threads waiting: #{@request_queue.num_waiting}" }
|
257
258
|
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
259
|
+
request_attempt = @request_queue.deq
|
260
|
+
|
261
|
+
if request_attempt.nil?
|
262
|
+
sleep(1)
|
263
|
+
else
|
264
|
+
request_attempt.attempted!
|
265
|
+
@requests_in_flight += 1
|
266
|
+
|
267
|
+
begin
|
268
|
+
resp = conn.request(request_attempt.request)
|
269
|
+
rescue => e
|
270
|
+
Timber::Config.instance.debug { "#deliver_requests error: #{e.message}" }
|
271
|
+
|
272
|
+
# Throw the request back on the queue for a retry if it has been attempted less
|
273
|
+
# than 3 times
|
274
|
+
if request_attempt.attempts < 3
|
275
|
+
Timber::Config.instance.debug { "Request is being retried, #{request_attempt.attempts} previous attempts" }
|
276
|
+
@request_queue.enq(request_attempt)
|
277
|
+
else
|
278
|
+
Timber::Config.instance.debug { "Request is being dropped, #{request_attempt.attempts} previous attempts" }
|
279
|
+
end
|
280
|
+
|
281
|
+
return false
|
282
|
+
ensure
|
283
|
+
@requests_in_flight -= 1
|
284
|
+
end
|
276
285
|
|
277
|
-
|
278
|
-
|
279
|
-
return false
|
280
|
-
ensure
|
281
|
-
@requests_in_flight -= 1
|
286
|
+
num_reqs += 1
|
287
|
+
Timber::Config.instance.debug { "Request successful: #{resp.code}" }
|
282
288
|
end
|
283
|
-
|
284
|
-
@successive_error_count = 0
|
285
|
-
num_reqs += 1
|
286
|
-
Timber::Config.instance.debug { "Request successful: #{resp.code}" }
|
287
289
|
end
|
290
|
+
|
291
|
+
true
|
288
292
|
end
|
289
293
|
|
290
294
|
# Builds the `Authorization` header value for HTTP delivery to the Timber API.
|
data/lib/timber/log_devices/http/{flushable_sized_queue.rb → flushable_dropping_sized_queue.rb}
RENAMED
@@ -2,11 +2,12 @@ module Timber
|
|
2
2
|
module LogDevices
|
3
3
|
class HTTP
|
4
4
|
# A simple thread-safe queue implementation that provides a #flush method.
|
5
|
-
# The built-in ruby Queue class does not provide a #flush method
|
6
|
-
#
|
7
|
-
#
|
5
|
+
# The built-in ruby `Queue` class does not provide a #flush method that allows
|
6
|
+
# the caller to retrieve all items on the queue in one call. The Ruby `SizedQueue` also
|
7
|
+
# implements thread waiting, which is something we want to avoid. To keep things
|
8
|
+
# simple and straight-forward, we designed this queue class.
|
8
9
|
# @private
|
9
|
-
class
|
10
|
+
class FlushableDroppingSizedQueue
|
10
11
|
def initialize(max_size)
|
11
12
|
@lock = Mutex.new
|
12
13
|
@max_size = max_size
|
@@ -14,9 +15,18 @@ module Timber
|
|
14
15
|
end
|
15
16
|
|
16
17
|
# Adds a message to the queue
|
17
|
-
def
|
18
|
+
def enq(msg)
|
18
19
|
@lock.synchronize do
|
19
|
-
|
20
|
+
if !full?
|
21
|
+
@array << msg
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
# Removes a single item from the queue
|
27
|
+
def deq
|
28
|
+
@lock.synchronize do
|
29
|
+
@array.pop
|
20
30
|
end
|
21
31
|
end
|
22
32
|
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Timber
|
2
|
+
module LogDevices
|
3
|
+
class HTTP
|
4
|
+
# Represents an attempt to deliver a request. Requests can be retried, hence
|
5
|
+
# why we keep track of the number of attempts.
|
6
|
+
class RequestAttempt
|
7
|
+
attr_reader :attempts, :request
|
8
|
+
|
9
|
+
def initialize(req)
|
10
|
+
@attempts = 0
|
11
|
+
@request = req
|
12
|
+
end
|
13
|
+
|
14
|
+
def attempted!
|
15
|
+
@attempts += 1
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
data/lib/timber/version.rb
CHANGED
@@ -100,9 +100,9 @@ describe Timber::LogDevices::HTTP do
|
|
100
100
|
http.write(log_entry)
|
101
101
|
http.send(:flush_async)
|
102
102
|
request_queue = http.instance_variable_get(:@request_queue)
|
103
|
-
|
104
|
-
expect(request).to be_kind_of(Net::HTTP::Post)
|
105
|
-
expect(request.body).to start_with("\x92\x84\xA5level\xA4INFO\xA2dt\xBB2016-09-01T12:00:00.000000Z\xA7message\xB2test log message 1".force_encoding("ASCII-8BIT"))
|
103
|
+
request_attempt = request_queue.deq
|
104
|
+
expect(request_attempt.request).to be_kind_of(Net::HTTP::Post)
|
105
|
+
expect(request_attempt.request.body).to start_with("\x92\x84\xA5level\xA4INFO\xA2dt\xBB2016-09-01T12:00:00.000000Z\xA7message\xB2test log message 1".force_encoding("ASCII-8BIT"))
|
106
106
|
|
107
107
|
message_queue = http.instance_variable_get(:@msg_queue)
|
108
108
|
expect(message_queue.size).to eq(0)
|
@@ -141,7 +141,7 @@ describe Timber::LogDevices::HTTP do
|
|
141
141
|
http.write(log_entry1)
|
142
142
|
log_entry2 = Timber::LogEntry.new("INFO", time, nil, "test log message 2", nil, nil)
|
143
143
|
http.write(log_entry2)
|
144
|
-
sleep
|
144
|
+
sleep 2
|
145
145
|
|
146
146
|
expect(stub).to have_been_requested.times(1)
|
147
147
|
|
@@ -157,8 +157,10 @@ describe Timber::LogDevices::HTTP do
|
|
157
157
|
req_queue = http_device.instance_variable_get(:@request_queue)
|
158
158
|
|
159
159
|
# Place a request on the queue
|
160
|
-
|
161
|
-
|
160
|
+
request = Net::HTTP::Post.new("/")
|
161
|
+
request_attempt = Timber::LogDevices::HTTP::RequestAttempt.new(request)
|
162
|
+
request_attempt.attempted!
|
163
|
+
req_queue.enq(request_attempt)
|
162
164
|
|
163
165
|
# Start a HTTP connection to test the method directly
|
164
166
|
http = http_device.send(:build_http)
|
@@ -168,6 +170,16 @@ describe Timber::LogDevices::HTTP do
|
|
168
170
|
end
|
169
171
|
|
170
172
|
expect(req_queue.size).to eq(1)
|
173
|
+
|
174
|
+
# Start a HTTP connection to test the method directly
|
175
|
+
http = http_device.send(:build_http)
|
176
|
+
http.start do |conn|
|
177
|
+
result = http_device.send(:deliver_requests, conn)
|
178
|
+
expect(result).to eq(false)
|
179
|
+
end
|
180
|
+
|
181
|
+
# Ensure the request gets discards after 3 attempts
|
182
|
+
expect(req_queue.size).to eq(0)
|
171
183
|
end
|
172
184
|
end
|
173
185
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: timber
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Timber Technologies, Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-09-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: msgpack
|
@@ -229,8 +229,8 @@ files:
|
|
229
229
|
- lib/timber/integrator.rb
|
230
230
|
- lib/timber/log_devices.rb
|
231
231
|
- lib/timber/log_devices/http.rb
|
232
|
-
- lib/timber/log_devices/http/
|
233
|
-
- lib/timber/log_devices/http/
|
232
|
+
- lib/timber/log_devices/http/flushable_dropping_sized_queue.rb
|
233
|
+
- lib/timber/log_devices/http/request_attempt.rb
|
234
234
|
- lib/timber/log_entry.rb
|
235
235
|
- lib/timber/logger.rb
|
236
236
|
- lib/timber/overrides.rb
|
@@ -1,26 +0,0 @@
|
|
1
|
-
module Timber
|
2
|
-
module LogDevices
|
3
|
-
class HTTP
|
4
|
-
# Works like SizedQueue, but drops message instead of blocking. Pass one of these in
|
5
|
-
# to {HTTP#intiialize} via the :request_queue option if you'd prefer to drop messages
|
6
|
-
# in the event of a buffer overflow instead of applying back pressure.
|
7
|
-
class DroppingSizedQueue < SizedQueue
|
8
|
-
# Returns true/false depending on whether the queue is full or not
|
9
|
-
def push(obj)
|
10
|
-
@mutex.synchronize do
|
11
|
-
return false unless @que.length < @max
|
12
|
-
|
13
|
-
@que.push obj
|
14
|
-
begin
|
15
|
-
t = @waiting.shift
|
16
|
-
t.wakeup if t
|
17
|
-
rescue ThreadError
|
18
|
-
retry
|
19
|
-
end
|
20
|
-
return true
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|