timber 2.1.10 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: e83f2fd0fc9e2e55f2a81089ac80ef335255b82f
4
- data.tar.gz: f860cc79667cc15469caf10600abfd9f4c5281a6
3
+ metadata.gz: d36593e9185855c949d8675ac749043317998c3c
4
+ data.tar.gz: 479e0355abf6e918aa1d0e0566e152c5fcfdaf97
5
5
  SHA512:
6
- metadata.gz: 81505965ac76980955b226ec83f5f43ba166b1ab671b5175a2a9f8f68e973bd7c320355c719f9b03792debe81ceac25c2e2832a517b1b931b31c28e8e31afb23
7
- data.tar.gz: b2a072f8ca6c6092c3e2ecff9040ca420b1fb32a0282b728f81d5bc90688619bb06e1062c2c11834bb10f91bd182c3d7755ecebdd591850eb0dd5ce04bb5c7b8
6
+ metadata.gz: 70397717e17995dcbf20b2d667037307a3f19c9702325442ea896e2d0c7c424adb4eea1bb7bba4e7b70d511acf11235018b639e2b7aef72b800e11965f6d0f8c
7
+ data.tar.gz: b4f322a1167c530eb1b5e0a90fc836781619a93d0f7a2e05ce4bcbb342bb53ef2356195c8df8ea5055f75cbeac63c19fb7598b891d6a87c8210e7c510f3b8378
@@ -1,5 +1,20 @@
1
1
  # Changelog
2
2
 
3
- Please see [https://github.com/timberio/timber-ruby/releases](https://github.com/timberio/timber-ruby/releases) for library specific changes.
3
+ All notable changes to this project will be documented in this file.
4
4
 
5
- For all Timber changes see [https://timber.io/changelog](https://timber.io/changelog).
5
+ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
6
+ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
7
+
8
+ ## [Unreleased]
9
+
10
+ ## [2.2.0] - 2017-09-13
11
+ ### Changed
12
+
13
+ - The default HTTP log device queue type was switched to a
14
+ `Timber::LogDevices::HTTP::FlushableDroppingSizedQueue` instead of a `::SizedQueue`. In the
15
+ event of extremely high volume logging, and delivery cannot keep up, Timber will drop messages
16
+ instead of applying back pressure.
17
+
18
+
19
+ [Unreleased]: https://github.com/timberio/timber-ruby/compare/v2.2.0...HEAD
20
+ [2.2.0]: https://github.com/timberio/timber-ruby/compare/v2.1.10...v2.2.0
@@ -3,8 +3,8 @@ require "msgpack"
3
3
  require "net/https"
4
4
 
5
5
  require "timber/config"
6
- require "timber/log_devices/http/dropping_sized_queue"
7
- require "timber/log_devices/http/flushable_sized_queue"
6
+ require "timber/log_devices/http/flushable_dropping_sized_queue"
7
+ require "timber/log_devices/http/request_attempt"
8
8
  require "timber/version"
9
9
 
10
10
  module Timber
@@ -52,10 +52,11 @@ module Timber
52
52
  # @option attributes [Symbol] :requests_per_conn (2500) The number of requests to send over a
53
53
  # single persistent connection. After this number is met, the connection will be closed
54
54
  # and a new one will be opened.
55
- # @option attributes [Symbol] :request_queue (SizedQueue.new(3)) The request queue object that queues Net::HTTP
56
- # requests for delivery. By deafult this is a `SizedQueue` of size `3`. Meaning once
57
- # 3 requests are placed on the queue for delivery, back pressure will be applied. IF
58
- # you'd prefer to drop messages instead, pass a {DroppingSizedQueue}. See examples for
55
+ # @option attributes [Symbol] :request_queue (FlushableDroppingSizedQueue.new(25)) The request
56
+ # queue object that queues Net::HTTP requests for delivery. By deafult this is a
57
+ # `FlushableDroppingSizedQueue` of size `25`. Meaning once the queue fills up to 25
58
+ # requests new requests will be dropped. If you'd prefer to apply back pressure,
59
+ # ensuring you do not lose log data, pass a standard {SizedQueue}. See examples for
59
60
  # an example.
60
61
  # @option attributes [Symbol] :timber_url The Timber URL to delivery the log lines. The
61
62
  # default is set via {TIMBER_URL}.
@@ -63,19 +64,18 @@ module Timber
63
64
  # @example Basic usage
64
65
  # Timber::Logger.new(Timber::LogDevices::HTTP.new("my_timber_api_key"))
65
66
  #
66
- # @example Dropping messages instead of applying back pressure
67
- # http_log_device = Timber::LogDevices::HTTP.new("my_timber_api_key",
68
- # request_queue: Timber::LogDevices::HTTP::DroppingSizedQueue.new(3))
67
+ # @example Apply back pressure instead of dropping messages
68
+ # http_log_device = Timber::LogDevices::HTTP.new("my_timber_api_key", request_queue: SizedQueue.new(25))
69
69
  # Timber::Logger.new(http_log_device)
70
70
  def initialize(api_key, options = {})
71
71
  @api_key = api_key || raise(ArgumentError.new("The api_key parameter cannot be blank"))
72
72
  @timber_url = URI.parse(options[:timber_url] || ENV['TIMBER_URL'] || TIMBER_URL)
73
73
  @batch_size = options[:batch_size] || 1_000
74
74
  @flush_continuously = options[:flush_continuously] != false
75
- @flush_interval = options[:flush_interval] || 1 # 1 second
75
+ @flush_interval = options[:flush_interval] || 2 # 2 seconds
76
76
  @requests_per_conn = options[:requests_per_conn] || 2_500
77
- @msg_queue = FlushableSizedQueue.new(@batch_size)
78
- @request_queue = options[:request_queue] || SizedQueue.new(3)
77
+ @msg_queue = FlushableDroppingSizedQueue.new(@batch_size)
78
+ @request_queue = options[:request_queue] || FlushableDroppingSizedQueue.new(25)
79
79
  @successive_error_count = 0
80
80
  @requests_in_flight = 0
81
81
  end
@@ -85,7 +85,7 @@ module Timber
85
85
  # size is constricted by the Timber API. The actual application limit is a multiple
86
86
  # of this. Hence the `@request_queue`.
87
87
  def write(msg)
88
- @msg_queue.enqueue(msg)
88
+ @msg_queue.enq(msg)
89
89
 
90
90
  # Lazily start flush threads to ensure threads are alive after forking processes.
91
91
  # If the threads are started during instantiation they will not be copied when
@@ -161,7 +161,8 @@ module Timber
161
161
  req = build_request
162
162
  if !req.nil?
163
163
  Timber::Config.instance.debug { "New request placed on queue" }
164
- @request_queue.enq(req)
164
+ request_attempt = RequestAttempt.new(req)
165
+ @request_queue.enq(request_attempt)
165
166
  end
166
167
  end
167
168
 
@@ -255,36 +256,39 @@ module Timber
255
256
  while num_reqs < @requests_per_conn
256
257
  Timber::Config.instance.debug { "Waiting on next request, threads waiting: #{@request_queue.num_waiting}" }
257
258
 
258
- # Blocks waiting for a request.
259
- req = @request_queue.deq
260
- @requests_in_flight += 1
261
-
262
- begin
263
- resp = conn.request(req)
264
- rescue => e
265
- Timber::Config.instance.debug { "#deliver_request error: #{e.message}" }
266
-
267
- @successive_error_count += 1
268
-
269
- # Back off so that we don't hammer the Timber API.
270
- calculated_backoff = @successive_error_count * 2
271
- backoff = calculated_backoff > 30 ? 30 : calculated_backoff
272
-
273
- Timber::Config.instance.debug { "Backing off #{backoff} seconds, error ##{@successive_error_count}" }
274
-
275
- sleep backoff
259
+ request_attempt = @request_queue.deq
260
+
261
+ if request_attempt.nil?
262
+ sleep(1)
263
+ else
264
+ request_attempt.attempted!
265
+ @requests_in_flight += 1
266
+
267
+ begin
268
+ resp = conn.request(request_attempt.request)
269
+ rescue => e
270
+ Timber::Config.instance.debug { "#deliver_requests error: #{e.message}" }
271
+
272
+ # Throw the request back on the queue for a retry if it has been attempted less
273
+ # than 3 times
274
+ if request_attempt.attempts < 3
275
+ Timber::Config.instance.debug { "Request is being retried, #{request_attempt.attempts} previous attempts" }
276
+ @request_queue.enq(request_attempt)
277
+ else
278
+ Timber::Config.instance.debug { "Request is being dropped, #{request_attempt.attempts} previous attempts" }
279
+ end
280
+
281
+ return false
282
+ ensure
283
+ @requests_in_flight -= 1
284
+ end
276
285
 
277
- # Throw the request back on the queue for a retry
278
- @request_queue.enq(req)
279
- return false
280
- ensure
281
- @requests_in_flight -= 1
286
+ num_reqs += 1
287
+ Timber::Config.instance.debug { "Request successful: #{resp.code}" }
282
288
  end
283
-
284
- @successive_error_count = 0
285
- num_reqs += 1
286
- Timber::Config.instance.debug { "Request successful: #{resp.code}" }
287
289
  end
290
+
291
+ true
288
292
  end
289
293
 
290
294
  # Builds the `Authorization` header value for HTTP delivery to the Timber API.
@@ -2,11 +2,12 @@ module Timber
2
2
  module LogDevices
3
3
  class HTTP
4
4
  # A simple thread-safe queue implementation that provides a #flush method.
5
- # The built-in ruby Queue class does not provide a #flush method. It also
6
- # implement thread waiting which is something we do not want. To keep things
7
- # simple and straight-forward we designed our own simple queue class.
5
+ # The built-in ruby `Queue` class does not provide a #flush method that allows
6
+ # the caller to retrieve all items on the queue in one call. The Ruby `SizedQueue` also
7
+ # implements thread waiting, which is something we want to avoid. To keep things
8
+ # simple and straight-forward, we designed this queue class.
8
9
  # @private
9
- class FlushableSizedQueue
10
+ class FlushableDroppingSizedQueue
10
11
  def initialize(max_size)
11
12
  @lock = Mutex.new
12
13
  @max_size = max_size
@@ -14,9 +15,18 @@ module Timber
14
15
  end
15
16
 
16
17
  # Adds a message to the queue
17
- def enqueue(msg)
18
+ def enq(msg)
18
19
  @lock.synchronize do
19
- @array << msg
20
+ if !full?
21
+ @array << msg
22
+ end
23
+ end
24
+ end
25
+
26
+ # Removes a single item from the queue
27
+ def deq
28
+ @lock.synchronize do
29
+ @array.pop
20
30
  end
21
31
  end
22
32
 
@@ -0,0 +1,20 @@
1
+ module Timber
2
+ module LogDevices
3
+ class HTTP
4
+ # Represents an attempt to deliver a request. Requests can be retried, hence
5
+ # why we keep track of the number of attempts.
6
+ class RequestAttempt
7
+ attr_reader :attempts, :request
8
+
9
+ def initialize(req)
10
+ @attempts = 0
11
+ @request = req
12
+ end
13
+
14
+ def attempted!
15
+ @attempts += 1
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -1,3 +1,3 @@
1
1
  module Timber
2
- VERSION = "2.1.10"
2
+ VERSION = "2.2.0"
3
3
  end
@@ -100,9 +100,9 @@ describe Timber::LogDevices::HTTP do
100
100
  http.write(log_entry)
101
101
  http.send(:flush_async)
102
102
  request_queue = http.instance_variable_get(:@request_queue)
103
- request = request_queue.deq
104
- expect(request).to be_kind_of(Net::HTTP::Post)
105
- expect(request.body).to start_with("\x92\x84\xA5level\xA4INFO\xA2dt\xBB2016-09-01T12:00:00.000000Z\xA7message\xB2test log message 1".force_encoding("ASCII-8BIT"))
103
+ request_attempt = request_queue.deq
104
+ expect(request_attempt.request).to be_kind_of(Net::HTTP::Post)
105
+ expect(request_attempt.request.body).to start_with("\x92\x84\xA5level\xA4INFO\xA2dt\xBB2016-09-01T12:00:00.000000Z\xA7message\xB2test log message 1".force_encoding("ASCII-8BIT"))
106
106
 
107
107
  message_queue = http.instance_variable_get(:@msg_queue)
108
108
  expect(message_queue.size).to eq(0)
@@ -141,7 +141,7 @@ describe Timber::LogDevices::HTTP do
141
141
  http.write(log_entry1)
142
142
  log_entry2 = Timber::LogEntry.new("INFO", time, nil, "test log message 2", nil, nil)
143
143
  http.write(log_entry2)
144
- sleep 1
144
+ sleep 2
145
145
 
146
146
  expect(stub).to have_been_requested.times(1)
147
147
 
@@ -157,8 +157,10 @@ describe Timber::LogDevices::HTTP do
157
157
  req_queue = http_device.instance_variable_get(:@request_queue)
158
158
 
159
159
  # Place a request on the queue
160
- req = Net::HTTP::Post.new("/")
161
- req_queue.enq(req)
160
+ request = Net::HTTP::Post.new("/")
161
+ request_attempt = Timber::LogDevices::HTTP::RequestAttempt.new(request)
162
+ request_attempt.attempted!
163
+ req_queue.enq(request_attempt)
162
164
 
163
165
  # Start a HTTP connection to test the method directly
164
166
  http = http_device.send(:build_http)
@@ -168,6 +170,16 @@ describe Timber::LogDevices::HTTP do
168
170
  end
169
171
 
170
172
  expect(req_queue.size).to eq(1)
173
+
174
+ # Start a HTTP connection to test the method directly
175
+ http = http_device.send(:build_http)
176
+ http.start do |conn|
177
+ result = http_device.send(:deliver_requests, conn)
178
+ expect(result).to eq(false)
179
+ end
180
+
181
+ # Ensure the request gets discards after 3 attempts
182
+ expect(req_queue.size).to eq(0)
171
183
  end
172
184
  end
173
185
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: timber
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.10
4
+ version: 2.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Timber Technologies, Inc.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-08-29 00:00:00.000000000 Z
11
+ date: 2017-09-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack
@@ -229,8 +229,8 @@ files:
229
229
  - lib/timber/integrator.rb
230
230
  - lib/timber/log_devices.rb
231
231
  - lib/timber/log_devices/http.rb
232
- - lib/timber/log_devices/http/dropping_sized_queue.rb
233
- - lib/timber/log_devices/http/flushable_sized_queue.rb
232
+ - lib/timber/log_devices/http/flushable_dropping_sized_queue.rb
233
+ - lib/timber/log_devices/http/request_attempt.rb
234
234
  - lib/timber/log_entry.rb
235
235
  - lib/timber/logger.rb
236
236
  - lib/timber/overrides.rb
@@ -1,26 +0,0 @@
1
- module Timber
2
- module LogDevices
3
- class HTTP
4
- # Works like SizedQueue, but drops message instead of blocking. Pass one of these in
5
- # to {HTTP#intiialize} via the :request_queue option if you'd prefer to drop messages
6
- # in the event of a buffer overflow instead of applying back pressure.
7
- class DroppingSizedQueue < SizedQueue
8
- # Returns true/false depending on whether the queue is full or not
9
- def push(obj)
10
- @mutex.synchronize do
11
- return false unless @que.length < @max
12
-
13
- @que.push obj
14
- begin
15
- t = @waiting.shift
16
- t.wakeup if t
17
- rescue ThreadError
18
- retry
19
- end
20
- return true
21
- end
22
- end
23
- end
24
- end
25
- end
26
- end