analytics-ruby 2.4.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/analytics +36 -36
- data/lib/analytics-ruby.rb +2 -0
- data/lib/segment/analytics/backoff_policy.rb +9 -15
- data/lib/segment/analytics/client.rb +12 -5
- data/lib/segment/analytics/defaults.rb +19 -3
- data/lib/segment/analytics/field_parser.rb +11 -6
- data/lib/segment/analytics/logging.rb +2 -0
- data/lib/segment/analytics/message_batch.rb +3 -0
- data/lib/segment/analytics/response.rb +6 -0
- data/lib/segment/analytics/retry_budget.rb +103 -0
- data/lib/segment/analytics/test_queue.rb +2 -0
- data/lib/segment/analytics/transport.rb +178 -60
- data/lib/segment/analytics/utils.rb +3 -1
- data/lib/segment/analytics/version.rb +3 -1
- data/lib/segment/analytics/worker.rb +3 -1
- data/lib/segment/analytics.rb +3 -0
- data/lib/segment.rb +2 -0
- metadata +19 -32
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 93b810e0ef6d4ef896010dfc80c9997308f0761ac2f79345d96fa434f7dd66a3
|
|
4
|
+
data.tar.gz: e69ffd005078968e7137d220d8f9e51249a5452ae5311bc37757b21ea9cf82c4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4619b6a5a332fd6da925f508c8e8ca58534236d597994ab0945840712d3761729a4c9dcf0a9aba2191e8f84f5191b853db9f1f854f047b00c46927ca98257c36
|
|
7
|
+
data.tar.gz: 9a6a6ca7a0c8c386dc9646c8521fb8000b183bd56d14d8da0a11e58117bd6beabd59048e8495f96b7acaab4624ad481c173dc3b781ca6dbbdcb24bd05d725a00
|
data/bin/analytics
CHANGED
|
@@ -43,62 +43,62 @@ command :send do |c|
|
|
|
43
43
|
|
|
44
44
|
c.action do |args, options|
|
|
45
45
|
Analytics = Segment::Analytics.new({
|
|
46
|
-
write_key
|
|
47
|
-
on_error
|
|
46
|
+
:write_key => options.writeKey,
|
|
47
|
+
:on_error => Proc.new { |status, msg| print msg }
|
|
48
48
|
})
|
|
49
49
|
|
|
50
50
|
case options.type
|
|
51
51
|
when "track"
|
|
52
52
|
Analytics.track({
|
|
53
|
-
user_id
|
|
54
|
-
event
|
|
55
|
-
anonymous_id
|
|
56
|
-
properties
|
|
57
|
-
context
|
|
58
|
-
integrations
|
|
53
|
+
:user_id => options.userId,
|
|
54
|
+
:event => options.event,
|
|
55
|
+
:anonymous_id => options.anonymousId,
|
|
56
|
+
:properties => json_hash(options.properties),
|
|
57
|
+
:context => json_hash(options.context),
|
|
58
|
+
:integrations => json_hash(options.integrations)
|
|
59
59
|
})
|
|
60
60
|
when "page"
|
|
61
61
|
Analytics.page({
|
|
62
|
-
user_id
|
|
63
|
-
anonymous_id
|
|
64
|
-
name
|
|
65
|
-
properties
|
|
66
|
-
context
|
|
67
|
-
integrations
|
|
62
|
+
:user_id => options.userId,
|
|
63
|
+
:anonymous_id => options.anonymousId,
|
|
64
|
+
:name => options.name,
|
|
65
|
+
:properties => json_hash(options.properties),
|
|
66
|
+
:context => json_hash(options.context),
|
|
67
|
+
:integrations => json_hash(options.integrations)
|
|
68
68
|
})
|
|
69
69
|
when "screen"
|
|
70
70
|
Analytics.screen({
|
|
71
|
-
user_id
|
|
72
|
-
anonymous_id
|
|
73
|
-
name
|
|
74
|
-
properties
|
|
75
|
-
context
|
|
76
|
-
integrations
|
|
71
|
+
:user_id => options.userId,
|
|
72
|
+
:anonymous_id => options.anonymousId,
|
|
73
|
+
:name => options.name,
|
|
74
|
+
:properties => json_hash(options.properties),
|
|
75
|
+
:context => json_hash(options.context),
|
|
76
|
+
:integrations => json_hash(options.integrations)
|
|
77
77
|
})
|
|
78
78
|
when "identify"
|
|
79
79
|
Analytics.identify({
|
|
80
|
-
user_id
|
|
81
|
-
anonymous_id
|
|
82
|
-
traits
|
|
83
|
-
context
|
|
84
|
-
integrations
|
|
80
|
+
:user_id => options.userId,
|
|
81
|
+
:anonymous_id => options.anonymousId,
|
|
82
|
+
:traits => json_hash(options.traits),
|
|
83
|
+
:context => json_hash(options.context),
|
|
84
|
+
:integrations => json_hash(options.integrations)
|
|
85
85
|
})
|
|
86
86
|
when "group"
|
|
87
87
|
Analytics.group({
|
|
88
|
-
user_id
|
|
89
|
-
anonymous_id
|
|
90
|
-
group_id
|
|
91
|
-
traits
|
|
92
|
-
context
|
|
93
|
-
integrations
|
|
88
|
+
:user_id => options.userId,
|
|
89
|
+
:anonymous_id => options.anonymousId,
|
|
90
|
+
:group_id => options.groupId,
|
|
91
|
+
:traits => json_hash(options.traits),
|
|
92
|
+
:context => json_hash(options.context),
|
|
93
|
+
:integrations => json_hash(options.integrations)
|
|
94
94
|
})
|
|
95
95
|
when "alias"
|
|
96
96
|
Analytics.alias({
|
|
97
|
-
previous_id
|
|
98
|
-
user_id
|
|
99
|
-
anonymous_id
|
|
100
|
-
context
|
|
101
|
-
integrations
|
|
97
|
+
:previous_id => options.previousId,
|
|
98
|
+
:user_id => options.userId,
|
|
99
|
+
:anonymous_id => options.anonymousId,
|
|
100
|
+
:context => json_hash(options.context),
|
|
101
|
+
:integrations => json_hash(options.integrations)
|
|
102
102
|
})
|
|
103
103
|
else
|
|
104
104
|
raise "Invalid Message Type #{options.type}"
|
data/lib/analytics-ruby.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'segment/analytics/defaults'
|
|
2
4
|
|
|
3
5
|
module Segment
|
|
@@ -24,25 +26,17 @@ module Segment
|
|
|
24
26
|
# @return [Numeric] the next backoff interval, in milliseconds.
|
|
25
27
|
def next_interval
|
|
26
28
|
interval = @min_timeout_ms * (@multiplier**@attempts)
|
|
27
|
-
interval = add_jitter(interval, @randomization_factor)
|
|
28
|
-
|
|
29
29
|
@attempts += 1
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
# Clamp first, then jitter. Jittering before the clamp meant every attempt
|
|
32
|
+
# at the ceiling returned exactly max_timeout_ms, so a fleet that backed off
|
|
33
|
+
# together stayed in lockstep. Jitter only subtracts, so the ceiling holds.
|
|
34
|
+
capped = [interval, @max_timeout_ms].min
|
|
35
|
+
capped - (rand * capped * @randomization_factor)
|
|
32
36
|
end
|
|
33
37
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def add_jitter(base, randomization_factor)
|
|
37
|
-
random_number = rand
|
|
38
|
-
max_deviation = base * randomization_factor
|
|
39
|
-
deviation = random_number * max_deviation
|
|
40
|
-
|
|
41
|
-
if random_number < 0.5
|
|
42
|
-
base - deviation
|
|
43
|
-
else
|
|
44
|
-
base + deviation
|
|
45
|
-
end
|
|
38
|
+
def reset!
|
|
39
|
+
@attempts = 0
|
|
46
40
|
end
|
|
47
41
|
end
|
|
48
42
|
end
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'thread'
|
|
2
4
|
require 'time'
|
|
3
5
|
|
|
@@ -30,7 +32,11 @@ module Segment
|
|
|
30
32
|
|
|
31
33
|
check_write_key!
|
|
32
34
|
|
|
33
|
-
|
|
35
|
+
# The worker checks this between sleep slices, so a Retry-After or backoff
|
|
36
|
+
# wait is abandoned within a second rather than holding shutdown for up to
|
|
37
|
+
# rate_limit_retry_after_cap seconds. Assigning to a dead thread is safe;
|
|
38
|
+
# Thread#wakeup is not, and raising here would force a non-zero exit status.
|
|
39
|
+
at_exit { @worker_thread[:should_exit] = true if @worker_thread }
|
|
34
40
|
end
|
|
35
41
|
|
|
36
42
|
# Synchronously waits until the worker has flushed the queue.
|
|
@@ -161,7 +167,10 @@ module Segment
|
|
|
161
167
|
# add our request id for tracing purposes
|
|
162
168
|
action[:messageId] ||= uid
|
|
163
169
|
|
|
164
|
-
|
|
170
|
+
if @test
|
|
171
|
+
test_queue << action
|
|
172
|
+
return true
|
|
173
|
+
end
|
|
165
174
|
|
|
166
175
|
if @queue.length < @max_queue_size
|
|
167
176
|
@queue << action
|
|
@@ -170,9 +179,7 @@ module Segment
|
|
|
170
179
|
true
|
|
171
180
|
else
|
|
172
181
|
logger.warn(
|
|
173
|
-
'Queue is full, dropping events. The :max_queue_size '
|
|
174
|
-
'configuration parameter can be increased to prevent this from ' \
|
|
175
|
-
'happening.'
|
|
182
|
+
'Queue is full, dropping events. The :max_queue_size configuration parameter can be increased to prevent this from happening.'
|
|
176
183
|
)
|
|
177
184
|
false
|
|
178
185
|
end
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module Segment
|
|
2
4
|
class Analytics
|
|
3
5
|
module Defaults
|
|
@@ -10,6 +12,20 @@ module Segment
|
|
|
10
12
|
'Content-Type' => 'application/json',
|
|
11
13
|
'User-Agent' => "analytics-ruby/#{Analytics::VERSION}" }
|
|
12
14
|
RETRIES = 10
|
|
15
|
+
MAX_TOTAL_BACKOFF_DURATION = 43_200 # 12 hours in seconds
|
|
16
|
+
|
|
17
|
+
# Rate-limited attempts are deliberately uncounted, so this duration is the
|
|
18
|
+
# only thing bounding them. It is deliberately several times
|
|
19
|
+
# RATE_LIMIT_RETRY_AFTER_CAP: when the two are equal a single maximal
|
|
20
|
+
# Retry-After consumes the whole budget, leaving one attempt and no retry,
|
|
21
|
+
# and the cap can never be the smaller of the two so it never binds at all.
|
|
22
|
+
MAX_RATE_LIMIT_DURATION = 1800 # seconds
|
|
23
|
+
|
|
24
|
+
# A guard against an absurd header, not a second budget. Waiting less than
|
|
25
|
+
# the server asked for does not make the next attempt more likely to
|
|
26
|
+
# succeed, it just sends more requests at something already rate-limiting
|
|
27
|
+
# us; how long we keep trying is MAX_RATE_LIMIT_DURATION's job.
|
|
28
|
+
RATE_LIMIT_RETRY_AFTER_CAP = 300 # seconds
|
|
13
29
|
end
|
|
14
30
|
|
|
15
31
|
module Queue
|
|
@@ -26,9 +42,9 @@ module Segment
|
|
|
26
42
|
end
|
|
27
43
|
|
|
28
44
|
module BackoffPolicy
|
|
29
|
-
MIN_TIMEOUT_MS =
|
|
30
|
-
MAX_TIMEOUT_MS =
|
|
31
|
-
MULTIPLIER =
|
|
45
|
+
MIN_TIMEOUT_MS = 500
|
|
46
|
+
MAX_TIMEOUT_MS = 60_000
|
|
47
|
+
MULTIPLIER = 2
|
|
32
48
|
RANDOMIZATION_FACTOR = 0.5
|
|
33
49
|
end
|
|
34
50
|
end
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module Segment
|
|
2
4
|
class Analytics
|
|
3
5
|
# Handles parsing fields according to the Segment Spec
|
|
@@ -160,9 +162,10 @@ module Segment
|
|
|
160
162
|
end
|
|
161
163
|
|
|
162
164
|
def check_user_id!(fields)
|
|
163
|
-
unless fields[:user_id]
|
|
164
|
-
|
|
165
|
-
|
|
165
|
+
return unless blank?(fields[:user_id])
|
|
166
|
+
return unless blank?(fields[:anonymous_id])
|
|
167
|
+
|
|
168
|
+
raise ArgumentError, 'Must supply either user_id or anonymous_id'
|
|
166
169
|
end
|
|
167
170
|
|
|
168
171
|
def check_timestamp!(timestamp)
|
|
@@ -178,9 +181,11 @@ module Segment
|
|
|
178
181
|
# obj - String|Number that must be non-blank
|
|
179
182
|
# name - Name of the validated value
|
|
180
183
|
def check_presence!(obj, name)
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
+
raise ArgumentError, "#{name} must be given" if blank?(obj)
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def blank?(obj)
|
|
188
|
+
obj.nil? || (obj.is_a?(String) && obj.empty?)
|
|
184
189
|
end
|
|
185
190
|
|
|
186
191
|
def check_is_hash!(obj, name)
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'forwardable'
|
|
2
4
|
require 'segment/analytics/logging'
|
|
3
5
|
|
|
@@ -27,6 +29,7 @@ module Segment
|
|
|
27
29
|
message_json_size = message_json.bytesize
|
|
28
30
|
if message_too_big?(message_json_size)
|
|
29
31
|
logger.error('a message exceeded the maximum allowed size')
|
|
32
|
+
raise JSONGenerationError, 'Message Exceeded Maximum Allowed Size'
|
|
30
33
|
else
|
|
31
34
|
@messages << message
|
|
32
35
|
@json_size += message_json_size + 1 # One byte for the comma
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Segment
|
|
4
|
+
class Analytics
|
|
5
|
+
# Tracks the two independent budgets one send may spend.
|
|
6
|
+
#
|
|
7
|
+
# A retryable status carrying Retry-After spends the rate-limit budget, which
|
|
8
|
+
# is bounded by wall clock only. Anything else retryable spends the counted
|
|
9
|
+
# backoff budget, bounded by both a retry count and wall clock. Keeping them
|
|
10
|
+
# separate is what stops a rate-limited server from exhausting the retries
|
|
11
|
+
# available to genuine failures.
|
|
12
|
+
#
|
|
13
|
+
# The caller performs the wait, so both methods return the delay in seconds,
|
|
14
|
+
# or nil when the budget is spent and the batch should be abandoned.
|
|
15
|
+
class RetryBudget
|
|
16
|
+
attr_reader :retry_count
|
|
17
|
+
|
|
18
|
+
# Keyword arguments would be cleaner but need Ruby 2.1; the gemspec still
|
|
19
|
+
# declares >= 2.0, which is also what rubocop is configured to parse.
|
|
20
|
+
def initialize(options = {})
|
|
21
|
+
@retries_remaining = options[:retries]
|
|
22
|
+
@backoff_policy = options[:backoff_policy]
|
|
23
|
+
@max_total_backoff_duration = options[:max_total_backoff_duration]
|
|
24
|
+
@max_rate_limit_duration = options[:max_rate_limit_duration]
|
|
25
|
+
@rate_limit_retry_after_cap = options[:rate_limit_retry_after_cap]
|
|
26
|
+
@logger = options[:logger]
|
|
27
|
+
@retry_count = 0
|
|
28
|
+
@backoff_start_time = nil
|
|
29
|
+
@rate_limit_start_time = nil
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def next_backoff_delay
|
|
33
|
+
# Checked before the decrement: decrementing first spent one retry on the
|
|
34
|
+
# exhaustion test itself, so a configured N only ever performed N-1, and
|
|
35
|
+
# retries: 1 and retries: 0 were indistinguishable.
|
|
36
|
+
return spent('Retries exhausted for batch') if @retries_remaining <= 0
|
|
37
|
+
|
|
38
|
+
@retries_remaining -= 1
|
|
39
|
+
|
|
40
|
+
# One reading for the budget test and the fit check below, as on the
|
|
41
|
+
# rate-limit path.
|
|
42
|
+
now = monotonic_now
|
|
43
|
+
@backoff_start_time ||= now
|
|
44
|
+
|
|
45
|
+
remaining = @max_total_backoff_duration - (now - @backoff_start_time)
|
|
46
|
+
return spent('Max total backoff duration exceeded for batch') if remaining <= 0
|
|
47
|
+
|
|
48
|
+
# A backoff longer than what is left of the budget is a wait whose attempt
|
|
49
|
+
# can never run, so there is nothing to schedule.
|
|
50
|
+
delay = @backoff_policy.next_interval.to_f / 1000
|
|
51
|
+
return spent('Backoff interval does not fit the remaining budget') if delay > remaining
|
|
52
|
+
|
|
53
|
+
@logger.debug("Retrying request, #{@retries_remaining} retries left. Waiting #{delay}s")
|
|
54
|
+
delay
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def next_rate_limit_delay(retry_after, status_code)
|
|
58
|
+
# One reading serves the episode start, the budget test and the delay. A
|
|
59
|
+
# second reading lets the budget expire between them, which yields a negative
|
|
60
|
+
# remaining and a negative delay — and Kernel#sleep raises ArgumentError on
|
|
61
|
+
# one rather than returning immediately. It also leaves remaining a hair under
|
|
62
|
+
# the budget on an episode's first response, which is enough to lose an exact
|
|
63
|
+
# comparison against the cap.
|
|
64
|
+
now = monotonic_now
|
|
65
|
+
@rate_limit_start_time ||= now
|
|
66
|
+
|
|
67
|
+
remaining = @max_rate_limit_duration - (now - @rate_limit_start_time)
|
|
68
|
+
return spent('Max rate limit duration exceeded for batch') if remaining <= 0
|
|
69
|
+
|
|
70
|
+
# Capped, then required to fit. Shortening the wait to fit would send the
|
|
71
|
+
# next request inside the window the server asked us to wait out — one it
|
|
72
|
+
# has already said it will not serve — and the budget is spent by then, so
|
|
73
|
+
# it would be the last attempt either way. Giving up loses the same batch
|
|
74
|
+
# and sends one request fewer at something already rate-limiting us.
|
|
75
|
+
delay = [retry_after, @rate_limit_retry_after_cap].min
|
|
76
|
+
return spent('Retry-After does not fit the remaining rate limit budget') if delay > remaining
|
|
77
|
+
|
|
78
|
+
@logger.debug("Retry-After: #{delay}s on #{status_code}. Retrying after delay.")
|
|
79
|
+
delay
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def record_retry
|
|
83
|
+
@retry_count += 1
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
private
|
|
87
|
+
|
|
88
|
+
def elapsed?(start_time, limit)
|
|
89
|
+
(monotonic_now - start_time) >= limit
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Wall-clock time can jump; these budgets must not expire or stretch with it.
|
|
93
|
+
def monotonic_now
|
|
94
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def spent(message)
|
|
98
|
+
@logger.error(message)
|
|
99
|
+
nil
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
@@ -1,11 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'segment/analytics/defaults'
|
|
2
4
|
require 'segment/analytics/utils'
|
|
3
5
|
require 'segment/analytics/response'
|
|
4
6
|
require 'segment/analytics/logging'
|
|
5
7
|
require 'segment/analytics/backoff_policy'
|
|
8
|
+
require 'segment/analytics/retry_budget'
|
|
6
9
|
require 'net/http'
|
|
7
10
|
require 'net/https'
|
|
8
11
|
require 'json'
|
|
12
|
+
require 'time'
|
|
9
13
|
|
|
10
14
|
module Segment
|
|
11
15
|
class Analytics
|
|
@@ -14,22 +18,18 @@ module Segment
|
|
|
14
18
|
include Segment::Analytics::Utils
|
|
15
19
|
include Segment::Analytics::Logging
|
|
16
20
|
|
|
21
|
+
RETRYABLE_4XX = [408, 410, 429, 460].freeze
|
|
22
|
+
NON_RETRYABLE_5XX = [501, 505, 511].freeze
|
|
23
|
+
|
|
17
24
|
def initialize(options = {})
|
|
18
25
|
options[:host] ||= HOST
|
|
19
26
|
options[:port] ||= PORT
|
|
20
|
-
options[:ssl]
|
|
27
|
+
options[:ssl] ||= SSL
|
|
21
28
|
@headers = options[:headers] || HEADERS
|
|
22
|
-
@path
|
|
23
|
-
@retries = options[:retries] || RETRIES
|
|
24
|
-
@backoff_policy =
|
|
25
|
-
options[:backoff_policy] || Segment::Analytics::BackoffPolicy.new
|
|
29
|
+
@path = options[:path] || PATH
|
|
26
30
|
|
|
27
|
-
|
|
28
|
-
http
|
|
29
|
-
http.read_timeout = 8
|
|
30
|
-
http.open_timeout = 4
|
|
31
|
-
|
|
32
|
-
@http = http
|
|
31
|
+
configure_retries(options)
|
|
32
|
+
@http = build_http(options)
|
|
33
33
|
end
|
|
34
34
|
|
|
35
35
|
# Sends a batch of messages to the API
|
|
@@ -38,23 +38,32 @@ module Segment
|
|
|
38
38
|
def send(write_key, batch)
|
|
39
39
|
logger.debug("Sending request for #{batch.length} items")
|
|
40
40
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
error = JSON.parse(body)['error']
|
|
44
|
-
should_retry = should_retry_request?(status_code, body)
|
|
45
|
-
logger.debug("Response status code: #{status_code}")
|
|
46
|
-
logger.debug("Response error: #{error}") if error
|
|
41
|
+
@backoff_policy.reset! if @backoff_policy.respond_to?(:reset!)
|
|
42
|
+
budget = new_retry_budget
|
|
47
43
|
|
|
48
|
-
|
|
49
|
-
|
|
44
|
+
loop do
|
|
45
|
+
begin
|
|
46
|
+
status_code, body, headers = send_request(write_key, batch, budget.retry_count)
|
|
47
|
+
rescue StandardError => e
|
|
48
|
+
# Connection reset, DNS failure, read timeout and friends. Retried on
|
|
49
|
+
# the counted backoff budget, like a retryable status code.
|
|
50
|
+
logger.error("Network error: #{e.message}")
|
|
51
|
+
return Response.new(-1, e.to_s) unless wait_to_retry(budget.next_backoff_delay, budget)
|
|
50
52
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
53
|
+
next
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
error = parse_error(body)
|
|
57
|
+
final = final_response(status_code, body, error)
|
|
58
|
+
return final if final
|
|
59
|
+
|
|
60
|
+
delay = retry_delay(status_code, headers, budget)
|
|
61
|
+
return Response.new(status_code, error) unless wait_to_retry(delay, budget)
|
|
57
62
|
end
|
|
63
|
+
rescue StandardError => e
|
|
64
|
+
logger.error(e.message)
|
|
65
|
+
e.backtrace.each { |line| logger.error(line) }
|
|
66
|
+
Response.new(-1, e.to_s)
|
|
58
67
|
end
|
|
59
68
|
|
|
60
69
|
# Closes a persistent connection if it exists
|
|
@@ -64,65 +73,174 @@ module Segment
|
|
|
64
73
|
|
|
65
74
|
private
|
|
66
75
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
76
|
+
# A Response once the batch is settled, or nil while it is still retryable.
|
|
77
|
+
def final_response(status_code, body, error)
|
|
78
|
+
logger.debug("Response status code: #{status_code}")
|
|
79
|
+
logger.debug("Response error: #{error}") if error
|
|
80
|
+
|
|
81
|
+
return Response.new(status_code, error) if success_status?(status_code)
|
|
82
|
+
return nil if retryable_status?(status_code)
|
|
83
|
+
|
|
84
|
+
if status_code >= 300 && status_code < 400
|
|
85
|
+
# Logging the body here would be useless: a redirect has none.
|
|
86
|
+
logger.error("Unexpected redirect (#{status_code}); batch not uploaded. " \
|
|
87
|
+
'Check whether the configured host points at a proxy or redirector.')
|
|
75
88
|
else
|
|
76
|
-
|
|
89
|
+
logger.error(body)
|
|
77
90
|
end
|
|
91
|
+
Response.new(status_code, error)
|
|
78
92
|
end
|
|
79
93
|
|
|
80
|
-
#
|
|
81
|
-
#
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
94
|
+
# A Retry-After spends the rate-limit budget; anything else retryable
|
|
95
|
+
# spends the counted backoff budget.
|
|
96
|
+
def retry_delay(status_code, headers, budget)
|
|
97
|
+
retry_after = parse_retry_after(headers['retry-after'])
|
|
98
|
+
return budget.next_backoff_delay unless retry_after
|
|
99
|
+
|
|
100
|
+
budget.next_rate_limit_delay(retry_after, status_code)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def configure_retries(options)
|
|
104
|
+
@retries = options[:retries] || RETRIES
|
|
105
|
+
@backoff_policy =
|
|
106
|
+
options[:backoff_policy] || Segment::Analytics::BackoffPolicy.new
|
|
107
|
+
|
|
108
|
+
# One policy instance serves every batch, so it has to be reset between
|
|
109
|
+
# them or attempt counts accumulate and each batch starts where the last
|
|
110
|
+
# one left off. reset! is not part of the older documented contract, which
|
|
111
|
+
# was next_interval alone, so a policy predating it still works — but it
|
|
112
|
+
# keeps that accumulation, and silently. Say so rather than letting an
|
|
113
|
+
# integrator find it as "retries get slower the longer we run".
|
|
114
|
+
unless @backoff_policy.respond_to?(:reset!)
|
|
115
|
+
logger.warn(
|
|
116
|
+
'backoff_policy does not implement reset!; attempt counts will ' \
|
|
117
|
+
'accumulate across batches. Add a reset! method that clears them.'
|
|
118
|
+
)
|
|
119
|
+
end
|
|
120
|
+
@max_total_backoff_duration = options[:max_total_backoff_duration] ||
|
|
121
|
+
MAX_TOTAL_BACKOFF_DURATION
|
|
122
|
+
@max_rate_limit_duration = options[:max_rate_limit_duration] ||
|
|
123
|
+
MAX_RATE_LIMIT_DURATION
|
|
124
|
+
@rate_limit_retry_after_cap = options[:rate_limit_retry_after_cap] ||
|
|
125
|
+
RATE_LIMIT_RETRY_AFTER_CAP
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def build_http(options)
|
|
129
|
+
http = Net::HTTP.new(options[:host], options[:port])
|
|
130
|
+
http.use_ssl = options[:ssl]
|
|
131
|
+
http.read_timeout = 8
|
|
132
|
+
http.open_timeout = 4
|
|
133
|
+
http
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# How long a sliced retry wait sleeps before re-checking for shutdown.
|
|
137
|
+
SHUTDOWN_CHECK_INTERVAL = 1
|
|
138
|
+
|
|
139
|
+
def new_retry_budget
|
|
140
|
+
RetryBudget.new(
|
|
141
|
+
:retries => @retries,
|
|
142
|
+
:backoff_policy => @backoff_policy,
|
|
143
|
+
:max_total_backoff_duration => @max_total_backoff_duration,
|
|
144
|
+
:max_rate_limit_duration => @max_rate_limit_duration,
|
|
145
|
+
:rate_limit_retry_after_cap => @rate_limit_retry_after_cap,
|
|
146
|
+
:logger => logger
|
|
147
|
+
)
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# nil delay means the budget is spent. Sleeping here rather than inside
|
|
151
|
+
# RetryBudget keeps the wait on Transport, where callers stub it.
|
|
152
|
+
def wait_to_retry(delay, budget)
|
|
153
|
+
return false if delay.nil?
|
|
154
|
+
return false unless interruptible_sleep(delay)
|
|
155
|
+
|
|
156
|
+
budget.record_retry
|
|
157
|
+
true
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# Sleeps in slices so shutdown is noticed within SHUTDOWN_CHECK_INTERVAL
|
|
161
|
+
# rather than after the whole delay, which can be rate_limit_retry_after_cap
|
|
162
|
+
# seconds. Returns false if shutdown was requested.
|
|
85
163
|
#
|
|
86
|
-
#
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
164
|
+
# Thread#wakeup is deliberately not used for this: it only cuts short a sleep
|
|
165
|
+
# already in progress, so a wakeup arriving while the worker is mid-request is
|
|
166
|
+
# lost and the next sleep still runs in full.
|
|
167
|
+
def interruptible_sleep(seconds)
|
|
168
|
+
remaining = seconds
|
|
169
|
+
while remaining > 0
|
|
170
|
+
return false if Thread.current[:should_exit]
|
|
90
171
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
rescue StandardError => e
|
|
95
|
-
should_retry = true
|
|
96
|
-
caught_exception = e
|
|
172
|
+
slice = [remaining, SHUTDOWN_CHECK_INTERVAL].min
|
|
173
|
+
sleep(slice)
|
|
174
|
+
remaining -= slice
|
|
97
175
|
end
|
|
176
|
+
!Thread.current[:should_exit]
|
|
177
|
+
end
|
|
98
178
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
179
|
+
def parse_error(body)
|
|
180
|
+
JSON.parse(body)['error']
|
|
181
|
+
rescue StandardError
|
|
182
|
+
nil
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# Only 2xx. Net::HTTP does not follow redirects, so a 3xx means nothing was
|
|
186
|
+
# uploaded; calling it success would drop the batch silently.
|
|
187
|
+
def success_status?(code)
|
|
188
|
+
code >= 200 && code < 300
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def retryable_status?(code)
|
|
192
|
+
if code >= 500 && code < 600
|
|
193
|
+
!NON_RETRYABLE_5XX.include?(code)
|
|
103
194
|
else
|
|
104
|
-
|
|
195
|
+
RETRYABLE_4XX.include?(code)
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def parse_retry_after(value)
|
|
200
|
+
return nil if value.nil?
|
|
201
|
+
|
|
202
|
+
str = value.is_a?(Array) ? value.first : value
|
|
203
|
+
return nil if str.nil?
|
|
204
|
+
|
|
205
|
+
str = str.strip
|
|
206
|
+
|
|
207
|
+
# Try integer seconds
|
|
208
|
+
if str =~ /\A\d+\z/
|
|
209
|
+
seconds = str.to_i
|
|
210
|
+
return seconds > 0 ? seconds : nil
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# Try HTTP-date (RFC 7231 S7.1.1.1)
|
|
214
|
+
begin
|
|
215
|
+
target = Time.httpdate(str)
|
|
216
|
+
seconds = (target - Time.now).to_i
|
|
217
|
+
seconds > 0 ? seconds : nil
|
|
218
|
+
rescue ArgumentError
|
|
219
|
+
nil
|
|
105
220
|
end
|
|
106
221
|
end
|
|
107
222
|
|
|
108
|
-
# Sends a request for the batch, returns [status_code, body]
|
|
109
|
-
def send_request(write_key, batch)
|
|
223
|
+
# Sends a request for the batch, returns [status_code, body, headers]
|
|
224
|
+
def send_request(write_key, batch, retry_count = 0)
|
|
110
225
|
payload = JSON.generate(
|
|
111
226
|
:sentAt => datetime_in_iso8601(Time.now),
|
|
112
227
|
:batch => batch
|
|
113
228
|
)
|
|
114
|
-
|
|
229
|
+
headers = @headers.dup
|
|
230
|
+
headers['X-Retry-Count'] = retry_count.to_s if retry_count > 0
|
|
231
|
+
|
|
232
|
+
request = Net::HTTP::Post.new(@path, headers)
|
|
115
233
|
request.basic_auth(write_key, nil)
|
|
116
234
|
|
|
117
235
|
if self.class.stub
|
|
118
236
|
logger.debug "stubbed request to #{@path}: " \
|
|
119
237
|
"write key = #{write_key}, batch = #{JSON.generate(batch)}"
|
|
120
238
|
|
|
121
|
-
[200, '{}']
|
|
239
|
+
[200, '{}', {}]
|
|
122
240
|
else
|
|
123
|
-
@http.start unless @http.started?
|
|
241
|
+
@http.start unless @http.started?
|
|
124
242
|
response = @http.request(request, payload)
|
|
125
|
-
[response.code.to_i, response.body]
|
|
243
|
+
[response.code.to_i, response.body, response.to_hash]
|
|
126
244
|
end
|
|
127
245
|
end
|
|
128
246
|
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'securerandom'
|
|
2
4
|
|
|
3
5
|
module Segment
|
|
@@ -65,7 +67,7 @@ module Segment
|
|
|
65
67
|
end
|
|
66
68
|
|
|
67
69
|
def time_in_iso8601(time)
|
|
68
|
-
"#{time.strftime('%Y-%m-%dT%H:%M:%S.%
|
|
70
|
+
"#{time.strftime('%Y-%m-%dT%H:%M:%S.%3N')}#{formatted_offset(time, true, 'Z')}"
|
|
69
71
|
end
|
|
70
72
|
|
|
71
73
|
def date_in_iso8601(date)
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'segment/analytics/defaults'
|
|
2
4
|
require 'segment/analytics/message_batch'
|
|
3
5
|
require 'segment/analytics/transport'
|
|
@@ -43,7 +45,7 @@ module Segment
|
|
|
43
45
|
end
|
|
44
46
|
|
|
45
47
|
res = @transport.send @write_key, @batch
|
|
46
|
-
@on_error.call(res.status, res.error) unless res.
|
|
48
|
+
@on_error.call(res.status, res.error) unless res.success?
|
|
47
49
|
|
|
48
50
|
@lock.synchronize { @batch.clear }
|
|
49
51
|
end
|
data/lib/segment/analytics.rb
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'segment/analytics/version'
|
|
2
4
|
require 'segment/analytics/defaults'
|
|
3
5
|
require 'segment/analytics/utils'
|
|
4
6
|
require 'segment/analytics/field_parser'
|
|
5
7
|
require 'segment/analytics/client'
|
|
6
8
|
require 'segment/analytics/worker'
|
|
9
|
+
require 'segment/analytics/retry_budget'
|
|
7
10
|
require 'segment/analytics/transport'
|
|
8
11
|
require 'segment/analytics/response'
|
|
9
12
|
require 'segment/analytics/logging'
|
data/lib/segment.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: analytics-ruby
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.
|
|
4
|
+
version: 2.6.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Segment.io
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2026-09-28 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: commander
|
|
@@ -30,14 +30,14 @@ dependencies:
|
|
|
30
30
|
requirements:
|
|
31
31
|
- - "~>"
|
|
32
32
|
- !ruby/object:Gem::Version
|
|
33
|
-
version: '
|
|
33
|
+
version: '13.0'
|
|
34
34
|
type: :development
|
|
35
35
|
prerelease: false
|
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
37
37
|
requirements:
|
|
38
38
|
- - "~>"
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
|
-
version: '
|
|
40
|
+
version: '13.0'
|
|
41
41
|
- !ruby/object:Gem::Dependency
|
|
42
42
|
name: rspec
|
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -56,72 +56,58 @@ dependencies:
|
|
|
56
56
|
name: tzinfo
|
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
|
58
58
|
requirements:
|
|
59
|
-
- -
|
|
59
|
+
- - "~>"
|
|
60
60
|
- !ruby/object:Gem::Version
|
|
61
|
-
version: 1.2
|
|
61
|
+
version: '1.2'
|
|
62
62
|
type: :development
|
|
63
63
|
prerelease: false
|
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
|
65
65
|
requirements:
|
|
66
|
-
- -
|
|
66
|
+
- - "~>"
|
|
67
67
|
- !ruby/object:Gem::Version
|
|
68
|
-
version: 1.2
|
|
68
|
+
version: '1.2'
|
|
69
69
|
- !ruby/object:Gem::Dependency
|
|
70
70
|
name: activesupport
|
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
|
72
72
|
requirements:
|
|
73
73
|
- - "~>"
|
|
74
74
|
- !ruby/object:Gem::Version
|
|
75
|
-
version:
|
|
75
|
+
version: 5.2.0
|
|
76
76
|
type: :development
|
|
77
77
|
prerelease: false
|
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
|
79
79
|
requirements:
|
|
80
80
|
- - "~>"
|
|
81
81
|
- !ruby/object:Gem::Version
|
|
82
|
-
version:
|
|
82
|
+
version: 5.2.0
|
|
83
83
|
- !ruby/object:Gem::Dependency
|
|
84
84
|
name: oj
|
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
|
86
86
|
requirements:
|
|
87
87
|
- - "~>"
|
|
88
88
|
- !ruby/object:Gem::Version
|
|
89
|
-
version: 3.6
|
|
89
|
+
version: '3.6'
|
|
90
90
|
type: :development
|
|
91
91
|
prerelease: false
|
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
|
93
93
|
requirements:
|
|
94
94
|
- - "~>"
|
|
95
95
|
- !ruby/object:Gem::Version
|
|
96
|
-
version: 3.6
|
|
96
|
+
version: '3.6'
|
|
97
97
|
- !ruby/object:Gem::Dependency
|
|
98
98
|
name: rubocop
|
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
|
100
100
|
requirements:
|
|
101
101
|
- - "~>"
|
|
102
102
|
- !ruby/object:Gem::Version
|
|
103
|
-
version:
|
|
104
|
-
type: :development
|
|
105
|
-
prerelease: false
|
|
106
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
107
|
-
requirements:
|
|
108
|
-
- - "~>"
|
|
109
|
-
- !ruby/object:Gem::Version
|
|
110
|
-
version: 0.51.0
|
|
111
|
-
- !ruby/object:Gem::Dependency
|
|
112
|
-
name: codecov
|
|
113
|
-
requirement: !ruby/object:Gem::Requirement
|
|
114
|
-
requirements:
|
|
115
|
-
- - "~>"
|
|
116
|
-
- !ruby/object:Gem::Version
|
|
117
|
-
version: 0.1.4
|
|
103
|
+
version: '1.0'
|
|
118
104
|
type: :development
|
|
119
105
|
prerelease: false
|
|
120
106
|
version_requirements: !ruby/object:Gem::Requirement
|
|
121
107
|
requirements:
|
|
122
108
|
- - "~>"
|
|
123
109
|
- !ruby/object:Gem::Version
|
|
124
|
-
version:
|
|
110
|
+
version: '1.0'
|
|
125
111
|
description: The Segment.io ruby analytics library
|
|
126
112
|
email: friends@segment.io
|
|
127
113
|
executables:
|
|
@@ -140,6 +126,7 @@ files:
|
|
|
140
126
|
- lib/segment/analytics/logging.rb
|
|
141
127
|
- lib/segment/analytics/message_batch.rb
|
|
142
128
|
- lib/segment/analytics/response.rb
|
|
129
|
+
- lib/segment/analytics/retry_budget.rb
|
|
143
130
|
- lib/segment/analytics/test_queue.rb
|
|
144
131
|
- lib/segment/analytics/transport.rb
|
|
145
132
|
- lib/segment/analytics/utils.rb
|
|
@@ -149,7 +136,7 @@ homepage: https://github.com/segmentio/analytics-ruby
|
|
|
149
136
|
licenses:
|
|
150
137
|
- MIT
|
|
151
138
|
metadata: {}
|
|
152
|
-
post_install_message:
|
|
139
|
+
post_install_message:
|
|
153
140
|
rdoc_options: []
|
|
154
141
|
require_paths:
|
|
155
142
|
- lib
|
|
@@ -164,8 +151,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
164
151
|
- !ruby/object:Gem::Version
|
|
165
152
|
version: '0'
|
|
166
153
|
requirements: []
|
|
167
|
-
rubygems_version: 3.
|
|
168
|
-
signing_key:
|
|
154
|
+
rubygems_version: 3.5.22
|
|
155
|
+
signing_key:
|
|
169
156
|
specification_version: 4
|
|
170
157
|
summary: Segment.io analytics library
|
|
171
158
|
test_files: []
|