sumologic-metrics 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +3 -0
- data/.travis.yml +10 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +36 -0
- data/LICENSE.txt +21 -0
- data/README.md +54 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/lib/sumologic/metrics/backoff_policy.rb +47 -0
- data/lib/sumologic/metrics/client.rb +124 -0
- data/lib/sumologic/metrics/defaults.rb +29 -0
- data/lib/sumologic/metrics/logging.rb +28 -0
- data/lib/sumologic/metrics/metric.rb +21 -0
- data/lib/sumologic/metrics/metric_batch.rb +60 -0
- data/lib/sumologic/metrics/request.rb +128 -0
- data/lib/sumologic/metrics/response.rb +13 -0
- data/lib/sumologic/metrics/utils.rb +29 -0
- data/lib/sumologic/metrics/version.rb +5 -0
- data/lib/sumologic/metrics/worker.rb +59 -0
- data/lib/sumologic/metrics.rb +42 -0
- data/lib/sumologic-metrics.rb +1 -0
- data/sumologic-metrics.gemspec +26 -0
- metadata +111 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 9348d8d0deeb5aaee506aaa542e6e3764a23944b18b7116cc500e70a61daf749
|
4
|
+
data.tar.gz: dab1c3be242d91ec6541ef085f2232443e506f9feb797196cd0483c28f00c398
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 22b57078ddb8b816387fac0b7305e4e647049471863189e7a5a55910914420b400a30e1e16b6b4d368af37d9be2508ca1abfd0966de7da556ce8376471616356
|
7
|
+
data.tar.gz: cd15cdc787769c4380eba58da16145a6e86942c616c0d75357bf607bae2120720ff22380eb9c9e5221f9ced61765d7df47417a955fa061d7f2fe655e09970ea5
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/CODE_OF_CONDUCT.md
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
# Contributor Covenant Code of Conduct
|
2
|
+
|
3
|
+
## Our Pledge
|
4
|
+
|
5
|
+
In the interest of fostering an open and welcoming environment, we as
|
6
|
+
contributors and maintainers pledge to making participation in our project and
|
7
|
+
our community a harassment-free experience for everyone, regardless of age, body
|
8
|
+
size, disability, ethnicity, gender identity and expression, level of experience,
|
9
|
+
nationality, personal appearance, race, religion, or sexual identity and
|
10
|
+
orientation.
|
11
|
+
|
12
|
+
## Our Standards
|
13
|
+
|
14
|
+
Examples of behavior that contributes to creating a positive environment
|
15
|
+
include:
|
16
|
+
|
17
|
+
* Using welcoming and inclusive language
|
18
|
+
* Being respectful of differing viewpoints and experiences
|
19
|
+
* Gracefully accepting constructive criticism
|
20
|
+
* Focusing on what is best for the community
|
21
|
+
* Showing empathy towards other community members
|
22
|
+
|
23
|
+
Examples of unacceptable behavior by participants include:
|
24
|
+
|
25
|
+
* The use of sexualized language or imagery and unwelcome sexual attention or
|
26
|
+
advances
|
27
|
+
* Trolling, insulting/derogatory comments, and personal or political attacks
|
28
|
+
* Public or private harassment
|
29
|
+
* Publishing others' private information, such as a physical or electronic
|
30
|
+
address, without explicit permission
|
31
|
+
* Other conduct which could reasonably be considered inappropriate in a
|
32
|
+
professional setting
|
33
|
+
|
34
|
+
## Our Responsibilities
|
35
|
+
|
36
|
+
Project maintainers are responsible for clarifying the standards of acceptable
|
37
|
+
behavior and are expected to take appropriate and fair corrective action in
|
38
|
+
response to any instances of unacceptable behavior.
|
39
|
+
|
40
|
+
Project maintainers have the right and responsibility to remove, edit, or
|
41
|
+
reject comments, commits, code, wiki edits, issues, and other contributions
|
42
|
+
that are not aligned to this Code of Conduct, or to ban temporarily or
|
43
|
+
permanently any contributor for other behaviors that they deem inappropriate,
|
44
|
+
threatening, offensive, or harmful.
|
45
|
+
|
46
|
+
## Scope
|
47
|
+
|
48
|
+
This Code of Conduct applies both within project spaces and in public spaces
|
49
|
+
when an individual is representing the project or its community. Examples of
|
50
|
+
representing a project or community include using an official project e-mail
|
51
|
+
address, posting via an official social media account, or acting as an appointed
|
52
|
+
representative at an online or offline event. Representation of a project may be
|
53
|
+
further defined and clarified by project maintainers.
|
54
|
+
|
55
|
+
## Enforcement
|
56
|
+
|
57
|
+
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
58
|
+
reported by contacting the project team at josacar@users.noreply.github.com. All
|
59
|
+
complaints will be reviewed and investigated and will result in a response that
|
60
|
+
is deemed necessary and appropriate to the circumstances. The project team is
|
61
|
+
obligated to maintain confidentiality with regard to the reporter of an incident.
|
62
|
+
Further details of specific enforcement policies may be posted separately.
|
63
|
+
|
64
|
+
Project maintainers who do not follow or enforce the Code of Conduct in good
|
65
|
+
faith may face temporary or permanent repercussions as determined by other
|
66
|
+
members of the project's leadership.
|
67
|
+
|
68
|
+
## Attribution
|
69
|
+
|
70
|
+
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
71
|
+
available at [http://contributor-covenant.org/version/1/4][version]
|
72
|
+
|
73
|
+
[homepage]: http://contributor-covenant.org
|
74
|
+
[version]: http://contributor-covenant.org/version/1/4/
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
sumologic-metrics (0.1.0)
|
5
|
+
|
6
|
+
GEM
|
7
|
+
remote: https://rubygems.org/
|
8
|
+
specs:
|
9
|
+
diff-lcs (1.3)
|
10
|
+
rake (11.3.0)
|
11
|
+
rspec (3.7.0)
|
12
|
+
rspec-core (~> 3.7.0)
|
13
|
+
rspec-expectations (~> 3.7.0)
|
14
|
+
rspec-mocks (~> 3.7.0)
|
15
|
+
rspec-core (3.7.1)
|
16
|
+
rspec-support (~> 3.7.0)
|
17
|
+
rspec-expectations (3.7.0)
|
18
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
19
|
+
rspec-support (~> 3.7.0)
|
20
|
+
rspec-mocks (3.7.0)
|
21
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
22
|
+
rspec-support (~> 3.7.0)
|
23
|
+
rspec-support (3.7.1)
|
24
|
+
|
25
|
+
PLATFORMS
|
26
|
+
java
|
27
|
+
ruby
|
28
|
+
|
29
|
+
DEPENDENCIES
|
30
|
+
bundler (~> 1.16)
|
31
|
+
rake (~> 11.0)
|
32
|
+
rspec (~> 3.7)
|
33
|
+
sumologic-metrics!
|
34
|
+
|
35
|
+
BUNDLED WITH
|
36
|
+
1.16.2
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2018 Jose Luis Salas
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
# Sumologic::Metrics
|
2
|
+
|
3
|
+
[![Build Status](https://www.travis-ci.org/josacar/sumologic-metrics.svg?branch=master)](https://www.travis-ci.org/josacar/sumologic-metrics)
|
4
|
+
|
5
|
+
Upload metrics to Sumologic!
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'sumologic-metrics'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install sumologic-metrics
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
```ruby
|
26
|
+
require 'sumologic-metrics'
|
27
|
+
|
28
|
+
url = ENV.fetch('SUMOLOGIC_ENDPOINT')
|
29
|
+
|
30
|
+
metrics = Sumologic::Metrics.new(collector_uri: url)
|
31
|
+
metrics.push('cluster=prod node=lb-1 metric=cpu ip=2.2.3.4 team=infra 99.12 1528020619')
|
32
|
+
```
|
33
|
+
|
34
|
+
## Development
|
35
|
+
|
36
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
37
|
+
|
38
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
39
|
+
|
40
|
+
## Contributing
|
41
|
+
|
42
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/josacar/sumologic-metrics. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
43
|
+
|
44
|
+
## License
|
45
|
+
|
46
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
47
|
+
|
48
|
+
## Code of Conduct
|
49
|
+
|
50
|
+
Everyone interacting in the Sumologic::Metrics project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/josacar/sumologic-metrics/blob/master/CODE_OF_CONDUCT.md).
|
51
|
+
|
52
|
+
## Thanks
|
53
|
+
|
54
|
+
- SegmentIO for their [analytics-ruby](https://github.com/segmentio/analytics-ruby) gem
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'sumologic/metrics'
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require 'irb'
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'sumologic/metrics/defaults'
|
2
|
+
|
3
|
+
module Sumologic; class Metrics
|
4
|
+
class BackoffPolicy
|
5
|
+
include Sumologic::Metrics::Defaults::BackoffPolicy
|
6
|
+
|
7
|
+
# @param [Hash] opts
|
8
|
+
# @option opts [Numeric] :min_timeout_ms The minimum backoff timeout
|
9
|
+
# @option opts [Numeric] :max_timeout_ms The maximum backoff timeout
|
10
|
+
# @option opts [Numeric] :multiplier The value to multiply the current
|
11
|
+
# interval with for each retry attempt
|
12
|
+
# @option opts [Numeric] :randomization_factor The randomization factor
|
13
|
+
# to use to create a range around the retry interval
|
14
|
+
def initialize(opts = {})
|
15
|
+
@min_timeout_ms = opts[:min_timeout_ms] || MIN_TIMEOUT_MS
|
16
|
+
@max_timeout_ms = opts[:max_timeout_ms] || MAX_TIMEOUT_MS
|
17
|
+
@multiplier = opts[:multiplier] || MULTIPLIER
|
18
|
+
@randomization_factor = opts[:randomization_factor] || RANDOMIZATION_FACTOR
|
19
|
+
|
20
|
+
@attempts = 0
|
21
|
+
end
|
22
|
+
|
23
|
+
# @return [Numeric] the next backoff interval, in milliseconds.
|
24
|
+
def next_interval
|
25
|
+
interval = @min_timeout_ms * (@multiplier**@attempts)
|
26
|
+
interval = add_jitter(interval, @randomization_factor)
|
27
|
+
|
28
|
+
@attempts += 1
|
29
|
+
|
30
|
+
[interval, @max_timeout_ms].min
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def add_jitter(base, randomization_factor)
|
36
|
+
random_number = rand
|
37
|
+
max_deviation = base * randomization_factor
|
38
|
+
deviation = random_number * max_deviation
|
39
|
+
|
40
|
+
if random_number < 0.5
|
41
|
+
base - deviation
|
42
|
+
else
|
43
|
+
base + deviation
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end; end
|
@@ -0,0 +1,124 @@
|
|
1
|
+
require 'thread'
|
2
|
+
require 'time'
|
3
|
+
|
4
|
+
require 'sumologic/metrics/defaults'
|
5
|
+
require 'sumologic/metrics/logging'
|
6
|
+
require 'sumologic/metrics/utils'
|
7
|
+
require 'sumologic/metrics/worker'
|
8
|
+
|
9
|
+
module Sumologic; class Metrics
|
10
|
+
class Client
|
11
|
+
include Sumologic::Metrics::Utils
|
12
|
+
include Sumologic::Metrics::Logging
|
13
|
+
|
14
|
+
# @param [Hash] opts
|
15
|
+
# @option opts [String] :collector_uri Your project's collector_uri
|
16
|
+
# @option opts [FixNum] :max_queue_size Maximum number of calls to be
|
17
|
+
# remain queued.
|
18
|
+
# @option opts [Proc] :on_error Handles error calls from the API.
|
19
|
+
def initialize(opts = {})
|
20
|
+
symbolize_keys!(opts)
|
21
|
+
|
22
|
+
@queue = Queue.new
|
23
|
+
@collector_uri = opts[:collector_uri]
|
24
|
+
@max_queue_size = opts[:max_queue_size] || Defaults::Queue::MAX_SIZE
|
25
|
+
@options = opts
|
26
|
+
@worker_mutex = Mutex.new
|
27
|
+
@worker = Worker.new(@queue, @collector_uri, @options)
|
28
|
+
|
29
|
+
check_collector_uri!
|
30
|
+
|
31
|
+
at_exit { @worker_thread && @worker_thread[:should_exit] = true }
|
32
|
+
end
|
33
|
+
|
34
|
+
# Synchronously waits until the worker has flushed the queue.
|
35
|
+
#
|
36
|
+
# Use only for scripts which are not long-running, and will specifically
|
37
|
+
# exit
|
38
|
+
def flush
|
39
|
+
while !@queue.empty? || @worker.is_requesting?
|
40
|
+
ensure_worker_running
|
41
|
+
sleep(0.1)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
# Pushes a metric
|
46
|
+
#
|
47
|
+
# @see https://help.sumologic.com/Send-Data/Sources/02Sources-for-Hosted-Collectors/HTTP-Source/Upload-Data-to-an-HTTP-Source#About_the_Carbon_2.0_example_data_points
|
48
|
+
#
|
49
|
+
# @param [String] metric
|
50
|
+
def push(metric)
|
51
|
+
check_presence!(metric)
|
52
|
+
check_carbon_format!(metric)
|
53
|
+
|
54
|
+
enqueue(metric)
|
55
|
+
end
|
56
|
+
|
57
|
+
# @return [Fixnum] number of metrics in the queue
|
58
|
+
def queued_metrics
|
59
|
+
@queue.length
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
|
64
|
+
# private: Enqueues the action.
|
65
|
+
#
|
66
|
+
# returns Boolean of whether the item was added to the queue.
|
67
|
+
def enqueue(metric)
|
68
|
+
if @queue.length < @max_queue_size
|
69
|
+
@queue << metric
|
70
|
+
ensure_worker_running
|
71
|
+
|
72
|
+
true
|
73
|
+
else
|
74
|
+
logger.warn(
|
75
|
+
'Queue is full, dropping events. The :max_queue_size ' \
|
76
|
+
'configuration parameter can be increased to prevent this from ' \
|
77
|
+
'happening.'
|
78
|
+
)
|
79
|
+
false
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
# private: Ensures that a string is non-empty
|
84
|
+
#
|
85
|
+
# obj - String|Number that must be non-blank
|
86
|
+
#
|
87
|
+
def check_presence!(obj)
|
88
|
+
if obj.nil? || (obj.is_a?(String) && obj.empty?)
|
89
|
+
raise ArgumentError, 'metric must be given'
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
# private: Checks that the collector_uri is properly initialized
|
94
|
+
def check_collector_uri!
|
95
|
+
raise ArgumentError, 'collector URI must be initialized' if @collector_uri.nil?
|
96
|
+
end
|
97
|
+
|
98
|
+
# private: Checks the metric is Carbon 2
|
99
|
+
def check_carbon_format!(metric)
|
100
|
+
tag = '[^\s]+=[^\s]+'
|
101
|
+
intrinsic_tags = "(?<intrinsic_tags>#{tag}(?: #{tag})+)"
|
102
|
+
meta_tags = "(?<meta_tags>#{tag}(?:\s#{tag})*)"
|
103
|
+
value = '(?<value>[\d|\.]+)'
|
104
|
+
timestamp = '(?<timestamp>\d+)'
|
105
|
+
r = Regexp.new(/#{intrinsic_tags} (?:#{meta_tags}\s)?#{value} #{timestamp}/)
|
106
|
+
raise ArgumentError, 'metric should be in Carbon 2 format' unless r.match?(metric)
|
107
|
+
end
|
108
|
+
|
109
|
+
|
110
|
+
def ensure_worker_running
|
111
|
+
return if worker_running?
|
112
|
+
@worker_mutex.synchronize do
|
113
|
+
return if worker_running?
|
114
|
+
@worker_thread = Thread.new do
|
115
|
+
@worker.run
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def worker_running?
|
121
|
+
@worker_thread && @worker_thread.alive?
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end; end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Sumologic; class Metrics; module Defaults
|
2
|
+
module Request
|
3
|
+
HEADERS = {
|
4
|
+
'Content-Type' => 'application/vnd.sumologic.carbon2',
|
5
|
+
'User-Agent' => "sumologic-metrics/#{Sumologic::Metrics::VERSION}"
|
6
|
+
}.freeze
|
7
|
+
RETRIES = 10
|
8
|
+
end
|
9
|
+
|
10
|
+
module Queue
|
11
|
+
MAX_SIZE = 10_000
|
12
|
+
end
|
13
|
+
|
14
|
+
module Metric
|
15
|
+
MAX_BYTES = 32_768 # 32Kb
|
16
|
+
end
|
17
|
+
|
18
|
+
module MetricBatch
|
19
|
+
MAX_BYTES = 512_000 # 500Kb
|
20
|
+
MAX_SIZE = 100
|
21
|
+
end
|
22
|
+
|
23
|
+
module BackoffPolicy
|
24
|
+
MIN_TIMEOUT_MS = 100
|
25
|
+
MAX_TIMEOUT_MS = 10_000
|
26
|
+
MULTIPLIER = 1.5
|
27
|
+
RANDOMIZATION_FACTOR = 0.5
|
28
|
+
end
|
29
|
+
end; end; end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
module Sumologic; class Metrics; module Logging
|
4
|
+
class << self
|
5
|
+
def logger
|
6
|
+
@logger ||= begin
|
7
|
+
logger = Logger.new(STDOUT)
|
8
|
+
logger.level = Logger::INFO
|
9
|
+
logger.progname = 'Sumologic::Metrics'
|
10
|
+
logger
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
attr_writer :logger
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.included(base)
|
18
|
+
class << base
|
19
|
+
def logger
|
20
|
+
Logging.logger
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def logger
|
26
|
+
Logging.logger
|
27
|
+
end
|
28
|
+
end; end; end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'sumologic/metrics/defaults'
|
2
|
+
|
3
|
+
module Sumologic; class Metrics
|
4
|
+
class Metric
|
5
|
+
def initialize(content)
|
6
|
+
@content = content
|
7
|
+
end
|
8
|
+
|
9
|
+
def too_big?
|
10
|
+
size > Defaults::Metric::MAX_BYTES
|
11
|
+
end
|
12
|
+
|
13
|
+
def size
|
14
|
+
@content.bytesize
|
15
|
+
end
|
16
|
+
|
17
|
+
def to_s
|
18
|
+
@content
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end; end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
require 'sumologic/metrics/logging'
|
3
|
+
|
4
|
+
module Sumologic; class Metrics
|
5
|
+
# A batch of `metric`s to be sent to the API
|
6
|
+
class MetricBatch
|
7
|
+
extend Forwardable
|
8
|
+
include Sumologic::Metrics::Logging
|
9
|
+
include Sumologic::Metrics::Defaults::MetricBatch
|
10
|
+
|
11
|
+
def initialize(max_metric_count)
|
12
|
+
@metrics = []
|
13
|
+
@max_metric_count = max_metric_count
|
14
|
+
@size = 0
|
15
|
+
end
|
16
|
+
|
17
|
+
def <<(metric)
|
18
|
+
if metric.too_big?
|
19
|
+
logger.error('a metric exceeded the maximum allowed size')
|
20
|
+
else
|
21
|
+
@metrics << metric
|
22
|
+
@size += metric.size + 2 # One byte for new line
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def full?
|
27
|
+
item_count_exhausted? || size_exhausted?
|
28
|
+
end
|
29
|
+
|
30
|
+
def clear
|
31
|
+
@metrics.clear
|
32
|
+
@json = 0
|
33
|
+
end
|
34
|
+
|
35
|
+
def to_s
|
36
|
+
@metrics.join("\n")
|
37
|
+
end
|
38
|
+
|
39
|
+
def_delegators :@metrics, :empty?
|
40
|
+
def_delegators :@metrics, :length
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def item_count_exhausted?
|
45
|
+
@metrics.length >= @max_metric_count
|
46
|
+
end
|
47
|
+
|
48
|
+
# We consider the max size here as just enough to leave room for one more
|
49
|
+
# metric of the largest size possible. This is a shortcut that allows us
|
50
|
+
# to use a native Ruby `Queue` that doesn't allow peeking. The tradeoff
|
51
|
+
# here is that we might fit in less metrics than possible into a batch.
|
52
|
+
#
|
53
|
+
# The alternative is to use our own `Queue` implementation that allows
|
54
|
+
# peeking, and to consider the next metric size when calculating whether
|
55
|
+
# the metric can be accomodated in this batch.
|
56
|
+
def size_exhausted?
|
57
|
+
@size >= (MAX_BYTES - Defaults::Metric::MAX_BYTES)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end; end
|
@@ -0,0 +1,128 @@
|
|
1
|
+
require 'sumologic/metrics/defaults'
|
2
|
+
require 'sumologic/metrics/utils'
|
3
|
+
require 'sumologic/metrics/response'
|
4
|
+
require 'sumologic/metrics/logging'
|
5
|
+
require 'sumologic/metrics/backoff_policy'
|
6
|
+
require 'net/http'
|
7
|
+
require 'net/https'
|
8
|
+
require 'uri'
|
9
|
+
|
10
|
+
module Sumologic; class Metrics
|
11
|
+
class Request
|
12
|
+
include Sumologic::Metrics::Defaults::Request
|
13
|
+
include Sumologic::Metrics::Utils
|
14
|
+
include Sumologic::Metrics::Logging
|
15
|
+
|
16
|
+
# public: Creates a new request object to send metrics batch
|
17
|
+
#
|
18
|
+
def initialize(options = {})
|
19
|
+
@headers = options[:headers] || HEADERS
|
20
|
+
@retries = options[:retries] || RETRIES
|
21
|
+
@backoff_policy =
|
22
|
+
options[:backoff_policy] || Sumologic::Metrics::BackoffPolicy.new
|
23
|
+
|
24
|
+
uri = URI(options.fetch(:uri))
|
25
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
26
|
+
http.set_debug_output(logger) if logger.level == Logger::DEBUG
|
27
|
+
http.use_ssl = uri.scheme == 'https'
|
28
|
+
http.read_timeout = 8
|
29
|
+
http.open_timeout = 4
|
30
|
+
|
31
|
+
@path = uri.path
|
32
|
+
@http = http
|
33
|
+
end
|
34
|
+
|
35
|
+
# public: Posts the write key and batch of messages to the API.
|
36
|
+
#
|
37
|
+
# returns - Response of the status and error if it exists
|
38
|
+
def post(batch)
|
39
|
+
last_response, exception = retry_with_backoff(@retries) do
|
40
|
+
status_code, body, message = send_request(batch)
|
41
|
+
|
42
|
+
should_retry = should_retry_request?(status_code, body)
|
43
|
+
|
44
|
+
[Response.new(status_code, message), should_retry]
|
45
|
+
end
|
46
|
+
|
47
|
+
if exception
|
48
|
+
logger.error(exception.message)
|
49
|
+
exception.backtrace.each { |line| logger.error(line) }
|
50
|
+
Response.new(-1, "Connection error: #{exception}")
|
51
|
+
else
|
52
|
+
last_response
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def should_retry_request?(status_code, body)
|
59
|
+
if status_code >= 500
|
60
|
+
true # Server error
|
61
|
+
elsif status_code == 429
|
62
|
+
true # Rate limited
|
63
|
+
elsif status_code >= 400
|
64
|
+
logger.error(body)
|
65
|
+
false # Client error. Do not retry, but log
|
66
|
+
else
|
67
|
+
false
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
# Takes a block that returns [result, should_retry].
|
72
|
+
#
|
73
|
+
# Retries upto `retries_remaining` times, if `should_retry` is false or
|
74
|
+
# an exception is raised. `@backoff_policy` is used to determine the
|
75
|
+
# duration to sleep between attempts
|
76
|
+
#
|
77
|
+
# Returns [last_result, raised_exception]
|
78
|
+
def retry_with_backoff(retries_remaining, &block)
|
79
|
+
result, caught_exception = nil
|
80
|
+
should_retry = false
|
81
|
+
|
82
|
+
begin
|
83
|
+
result, should_retry = yield
|
84
|
+
return [result, nil] unless should_retry
|
85
|
+
rescue StandardError => e
|
86
|
+
p e
|
87
|
+
should_retry = true
|
88
|
+
caught_exception = e
|
89
|
+
end
|
90
|
+
|
91
|
+
if should_retry && (retries_remaining > 1)
|
92
|
+
sleep(@backoff_policy.next_interval.to_f / 1000)
|
93
|
+
retry_with_backoff(retries_remaining - 1, &block)
|
94
|
+
else
|
95
|
+
[result, caught_exception]
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
# Sends a request for the batch, returns [status_code, body]
|
100
|
+
def send_request(batch)
|
101
|
+
payload = batch.to_s
|
102
|
+
request = Net::HTTP::Post.new(@path, @headers)
|
103
|
+
|
104
|
+
if self.class.stub
|
105
|
+
logger.debug "stubbed request to #{@path}: " \
|
106
|
+
"batch = '#{batch}'"
|
107
|
+
|
108
|
+
[200, '', 'OK']
|
109
|
+
else
|
110
|
+
# If `start` is not called, Ruby adds a 'Connection: close' header to
|
111
|
+
# all requests, preventing us from reusing a connection for multiple
|
112
|
+
# HTTP requests
|
113
|
+
@http.start unless @http.started?
|
114
|
+
|
115
|
+
response = @http.request(request, payload)
|
116
|
+
[response.code.to_i, response.body, response.message]
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
class << self
|
121
|
+
attr_writer :stub
|
122
|
+
|
123
|
+
def stub
|
124
|
+
@stub || ENV['STUB']
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end; end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module Sumologic; class Metrics
|
2
|
+
class Response
|
3
|
+
attr_reader :status, :message
|
4
|
+
|
5
|
+
# public: Simple class to wrap responses from the API
|
6
|
+
#
|
7
|
+
#
|
8
|
+
def initialize(status = 200, message = nil)
|
9
|
+
@status = status
|
10
|
+
@message = message
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end; end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'securerandom'
|
2
|
+
|
3
|
+
module Sumologic; class Metrics
|
4
|
+
module Utils
|
5
|
+
module_function
|
6
|
+
|
7
|
+
# public: Return a new hash with keys converted from strings to symbols
|
8
|
+
#
|
9
|
+
def symbolize_keys(hash)
|
10
|
+
hash.each_with_object({}) do |(k, v), memo|
|
11
|
+
memo[k.to_sym] = v
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
# public: Convert hash keys from strings to symbols in place
|
16
|
+
#
|
17
|
+
def symbolize_keys!(hash)
|
18
|
+
hash.replace(symbolize_keys(hash))
|
19
|
+
end
|
20
|
+
|
21
|
+
# public: Return a new hash with keys as strings
|
22
|
+
#
|
23
|
+
def stringify_keys(hash)
|
24
|
+
hash.each_with_object({}) do |(k, v), memo|
|
25
|
+
memo[k.to_s] = v
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end; end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
require 'sumologic/metrics/defaults'
|
2
|
+
require 'sumologic/metrics/metric'
|
3
|
+
require 'sumologic/metrics/metric_batch'
|
4
|
+
require 'sumologic/metrics/request'
|
5
|
+
require 'sumologic/metrics/utils'
|
6
|
+
|
7
|
+
module Sumologic; class Metrics
|
8
|
+
class Worker
|
9
|
+
include Sumologic::Metrics::Utils
|
10
|
+
include Sumologic::Metrics::Defaults
|
11
|
+
|
12
|
+
# public: Creates a new worker
|
13
|
+
#
|
14
|
+
# The worker continuously takes metrics off the queue
|
15
|
+
# and makes requests to the Sumologic api
|
16
|
+
#
|
17
|
+
# queue - Queue synchronized between client and worker
|
18
|
+
# collector_uri - String of the unique collector URI
|
19
|
+
# options - Hash of worker options
|
20
|
+
# batch_size - Fixnum of how many items to send in a batch
|
21
|
+
# on_error - Proc of what to do on an error
|
22
|
+
#
|
23
|
+
def initialize(queue, collector_uri, options = {})
|
24
|
+
symbolize_keys! options
|
25
|
+
@queue = queue
|
26
|
+
@collector_uri = collector_uri
|
27
|
+
@on_error = options[:on_error] || proc { |status, message| }
|
28
|
+
batch_size = options[:batch_size] || Defaults::MetricBatch::MAX_SIZE
|
29
|
+
@batch = MetricBatch.new(batch_size)
|
30
|
+
@lock = Mutex.new
|
31
|
+
@request = options[:request] || Request.new(uri: collector_uri)
|
32
|
+
end
|
33
|
+
|
34
|
+
# public: Continuously runs the loop to check for new events
|
35
|
+
#
|
36
|
+
def run
|
37
|
+
until Thread.current[:should_exit]
|
38
|
+
return if @queue.empty?
|
39
|
+
|
40
|
+
@lock.synchronize do
|
41
|
+
until @batch.full? || @queue.empty?
|
42
|
+
@batch << Metric.new(@queue.pop)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
res = @request.post(@batch)
|
47
|
+
@on_error.call(res.status, res.message) unless res.status == 200
|
48
|
+
|
49
|
+
@lock.synchronize { @batch.clear }
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
# public: Check whether we have outstanding requests.
|
54
|
+
#
|
55
|
+
def is_requesting?
|
56
|
+
@lock.synchronize { !@batch.empty? }
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end; end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'sumologic/metrics/backoff_policy'
|
2
|
+
require 'sumologic/metrics/client'
|
3
|
+
require 'sumologic/metrics/defaults'
|
4
|
+
require 'sumologic/metrics/metric'
|
5
|
+
require 'sumologic/metrics/metric_batch'
|
6
|
+
require 'sumologic/metrics/logging'
|
7
|
+
require 'sumologic/metrics/worker'
|
8
|
+
require 'sumologic/metrics/request'
|
9
|
+
require 'sumologic/metrics/response'
|
10
|
+
require 'sumologic/metrics/utils'
|
11
|
+
require 'sumologic/metrics/version'
|
12
|
+
require 'sumologic/metrics/worker'
|
13
|
+
|
14
|
+
module Sumologic
|
15
|
+
class Metrics
|
16
|
+
# Initializes a new instance of {Sumologic::Metrics::Client}, to which all
|
17
|
+
# method calls are proxied.
|
18
|
+
#
|
19
|
+
# @param options includes options that are passed down to
|
20
|
+
# {Sumologic::Metrics::Client#initialize}
|
21
|
+
# @option options [Boolean] :stub (false) If true, requests don't hit the
|
22
|
+
# server and are stubbed to be successful.
|
23
|
+
def initialize(options = {})
|
24
|
+
Request.stub = options[:stub] if options.key?(:stub)
|
25
|
+
@client = Sumologic::Metrics::Client.new(options)
|
26
|
+
end
|
27
|
+
|
28
|
+
def method_missing(message, *args, &block)
|
29
|
+
if @client.respond_to?(message)
|
30
|
+
@client.send(message, *args, &block)
|
31
|
+
else
|
32
|
+
super
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def respond_to_missing?(method_name, include_private = false)
|
37
|
+
@client.respond_to?(method_name) || super
|
38
|
+
end
|
39
|
+
|
40
|
+
include Logging
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
require 'sumologic/metrics'
|
@@ -0,0 +1,26 @@
|
|
1
|
+
lib = File.expand_path('lib', __dir__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
require 'sumologic/metrics/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = 'sumologic-metrics'
|
7
|
+
spec.version = Sumologic::Metrics::VERSION
|
8
|
+
spec.authors = ['Jose Luis Salas']
|
9
|
+
spec.email = ['josacar@users.noreply.github.com']
|
10
|
+
|
11
|
+
spec.summary = 'Sumologic metrics worker'
|
12
|
+
spec.description = 'Send metrics to Sumologic in the background in batches'
|
13
|
+
spec.homepage = 'https://github.com/josacar/sumologic-metrics'
|
14
|
+
spec.license = 'MIT'
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
17
|
+
f.match(%r{^(test|spec|features)/})
|
18
|
+
end
|
19
|
+
spec.bindir = 'exe'
|
20
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
|
+
spec.require_paths = ['lib']
|
22
|
+
|
23
|
+
spec.add_development_dependency 'bundler', '~> 1.16'
|
24
|
+
spec.add_development_dependency 'rake', '~> 11.0'
|
25
|
+
spec.add_development_dependency 'rspec', '~> 3.7'
|
26
|
+
end
|
metadata
ADDED
@@ -0,0 +1,111 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sumologic-metrics
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Jose Luis Salas
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2018-06-03 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.16'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.16'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '11.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '11.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.7'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.7'
|
55
|
+
description: Send metrics to Sumologic in the background in batches
|
56
|
+
email:
|
57
|
+
- josacar@users.noreply.github.com
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- ".gitignore"
|
63
|
+
- ".rspec"
|
64
|
+
- ".travis.yml"
|
65
|
+
- CODE_OF_CONDUCT.md
|
66
|
+
- Gemfile
|
67
|
+
- Gemfile.lock
|
68
|
+
- LICENSE.txt
|
69
|
+
- README.md
|
70
|
+
- Rakefile
|
71
|
+
- bin/console
|
72
|
+
- bin/setup
|
73
|
+
- lib/sumologic-metrics.rb
|
74
|
+
- lib/sumologic/metrics.rb
|
75
|
+
- lib/sumologic/metrics/backoff_policy.rb
|
76
|
+
- lib/sumologic/metrics/client.rb
|
77
|
+
- lib/sumologic/metrics/defaults.rb
|
78
|
+
- lib/sumologic/metrics/logging.rb
|
79
|
+
- lib/sumologic/metrics/metric.rb
|
80
|
+
- lib/sumologic/metrics/metric_batch.rb
|
81
|
+
- lib/sumologic/metrics/request.rb
|
82
|
+
- lib/sumologic/metrics/response.rb
|
83
|
+
- lib/sumologic/metrics/utils.rb
|
84
|
+
- lib/sumologic/metrics/version.rb
|
85
|
+
- lib/sumologic/metrics/worker.rb
|
86
|
+
- sumologic-metrics.gemspec
|
87
|
+
homepage: https://github.com/josacar/sumologic-metrics
|
88
|
+
licenses:
|
89
|
+
- MIT
|
90
|
+
metadata: {}
|
91
|
+
post_install_message:
|
92
|
+
rdoc_options: []
|
93
|
+
require_paths:
|
94
|
+
- lib
|
95
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
96
|
+
requirements:
|
97
|
+
- - ">="
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
version: '0'
|
100
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
101
|
+
requirements:
|
102
|
+
- - ">="
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
version: '0'
|
105
|
+
requirements: []
|
106
|
+
rubyforge_project:
|
107
|
+
rubygems_version: 2.7.6
|
108
|
+
signing_key:
|
109
|
+
specification_version: 4
|
110
|
+
summary: Sumologic metrics worker
|
111
|
+
test_files: []
|