qubole-statsd-instrument 2.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +7 -0
  3. data/.travis.yml +12 -0
  4. data/CHANGELOG.md +89 -0
  5. data/CONTRIBUTING.md +34 -0
  6. data/Gemfile +2 -0
  7. data/LICENSE +20 -0
  8. data/README.md +319 -0
  9. data/Rakefile +10 -0
  10. data/lib/statsd/instrument/assertions.rb +88 -0
  11. data/lib/statsd/instrument/backend.rb +17 -0
  12. data/lib/statsd/instrument/backends/capture_backend.rb +29 -0
  13. data/lib/statsd/instrument/backends/logger_backend.rb +20 -0
  14. data/lib/statsd/instrument/backends/null_backend.rb +7 -0
  15. data/lib/statsd/instrument/backends/udp_backend.rb +106 -0
  16. data/lib/statsd/instrument/environment.rb +54 -0
  17. data/lib/statsd/instrument/helpers.rb +14 -0
  18. data/lib/statsd/instrument/matchers.rb +96 -0
  19. data/lib/statsd/instrument/metric.rb +117 -0
  20. data/lib/statsd/instrument/metric_expectation.rb +67 -0
  21. data/lib/statsd/instrument/railtie.rb +14 -0
  22. data/lib/statsd/instrument/version.rb +5 -0
  23. data/lib/statsd/instrument.rb +407 -0
  24. data/lib/statsd-instrument.rb +1 -0
  25. data/shipit.rubygems.yml +1 -0
  26. data/statsd-instrument.gemspec +27 -0
  27. data/test/assertions_test.rb +329 -0
  28. data/test/benchmark/tags.rb +34 -0
  29. data/test/capture_backend_test.rb +24 -0
  30. data/test/environment_test.rb +46 -0
  31. data/test/helpers_test.rb +24 -0
  32. data/test/integration_test.rb +20 -0
  33. data/test/logger_backend_test.rb +20 -0
  34. data/test/matchers_test.rb +102 -0
  35. data/test/metric_test.rb +45 -0
  36. data/test/statsd_instrumentation_test.rb +328 -0
  37. data/test/statsd_test.rb +136 -0
  38. data/test/test_helper.rb +10 -0
  39. data/test/udp_backend_test.rb +167 -0
  40. metadata +182 -0
@@ -0,0 +1,106 @@
1
+ require 'monitor'
2
+
3
+ module StatsD::Instrument::Backends
4
+ class UDPBackend < StatsD::Instrument::Backend
5
+
6
+ DEFAULT_IMPLEMENTATION = :statsd
7
+
8
+ include MonitorMixin
9
+
10
+ attr_reader :host, :port
11
+ attr_accessor :implementation
12
+
13
+ def initialize(server = nil, implementation = nil)
14
+ super()
15
+ self.server = server || "localhost:8125"
16
+ self.implementation = (implementation || DEFAULT_IMPLEMENTATION).to_sym
17
+ end
18
+
19
+ def collect_metric(metric)
20
+ unless implementation_supports_metric_type?(metric.type)
21
+ StatsD.logger.warn("[StatsD] Metric type #{metric.type.inspect} not supported on #{implementation} implementation.")
22
+ return false
23
+ end
24
+
25
+ if metric.sample_rate < 1.0 && rand > metric.sample_rate
26
+ return false
27
+ end
28
+
29
+ write_packet(generate_packet(metric))
30
+ end
31
+
32
+ def implementation_supports_metric_type?(type)
33
+ case type
34
+ when :h; implementation == :datadog
35
+ when :kv; implementation == :statsite
36
+ else true
37
+ end
38
+ end
39
+
40
+ def server=(connection_string)
41
+ self.host, port = connection_string.split(':', 2)
42
+ self.port = port.to_i
43
+ invalidate_socket
44
+ end
45
+
46
+ def host=(host)
47
+ @host = host
48
+ invalidate_socket
49
+ end
50
+
51
+ def port=(port)
52
+ @port = port
53
+ invalidate_socket
54
+ end
55
+
56
+ def socket
57
+ if @socket.nil?
58
+ @socket = UDPSocket.new
59
+ @socket.connect(host, port)
60
+ end
61
+ @socket
62
+ end
63
+
64
+ def generate_packet(metric)
65
+ command = "#{metric.name}:#{metric.value}|#{metric.type}"
66
+ command << "|@#{metric.sample_rate}" if metric.sample_rate < 1 || (implementation == :statsite && metric.sample_rate > 1)
67
+ if metric.tags
68
+ if tags_supported?
69
+ if implementation == :datadog
70
+ command << "|##{metric.tags.join(',')}"
71
+ elsif implementation == :collectd
72
+ metric_tags = "#{metric.tags.join(',')}"
73
+ metric_tags = metric_tags.prepend("[") << "]"
74
+ command.prepend(metric_tags.gsub(":", "="))
75
+ end
76
+ else
77
+ StatsD.logger.warn("[StatsD] Tags are only supported on Datadog and CollectD implementation.")
78
+ end
79
+ end
80
+
81
+ command << "\n" if implementation == :statsite
82
+ command
83
+ end
84
+
85
+ def tags_supported?
86
+ implementation == :datadog || implementation == :collectd
87
+ end
88
+
89
+ def write_packet(command)
90
+ synchronize do
91
+ socket.send(command, 0) > 0
92
+ end
93
+ rescue ThreadError => e
94
+ # In cases where a TERM or KILL signal has been sent, and we send stats as
95
+ # part of a signal handler, locks cannot be acquired, so we do our best
96
+ # to try and send the command without a lock.
97
+ socket.send(command, 0) > 0
98
+ rescue SocketError, IOError, SystemCallError, Errno::ECONNREFUSED => e
99
+ StatsD.logger.error "[StatsD] #{e.class.name}: #{e.message}"
100
+ end
101
+
102
+ def invalidate_socket
103
+ @socket = nil
104
+ end
105
+ end
106
+ end
@@ -0,0 +1,54 @@
1
+ require 'logger'
2
+
3
+ # The environment module is used to detect, and initialize the environment in
4
+ # which this library is active. It will use different default values based on the environment.
5
+ module StatsD::Instrument::Environment
6
+ extend self
7
+
8
+ # Instantiates a default backend for the current environment.
9
+ #
10
+ # @return [StatsD::Instrument::Backend]
11
+ # @see #environment
12
+ def default_backend
13
+ case environment
14
+ when 'production', 'staging'
15
+ StatsD::Instrument::Backends::UDPBackend.new(ENV['STATSD_ADDR'], ENV['STATSD_IMPLEMENTATION'])
16
+ when 'test'
17
+ StatsD::Instrument::Backends::NullBackend.new
18
+ else
19
+ StatsD::Instrument::Backends::LoggerBackend.new(StatsD.logger)
20
+ end
21
+ end
22
+
23
+ # Detects the current environment, either by asking Rails, or by inspecting environment variables.
24
+ #
25
+ # - Within a Rails application, <tt>Rails.env</tt> is used.
26
+ # - It will check the following environment variables in order: <tt>RAILS_ENV</tt>, <tt>RACK_ENV</tt>, <tt>ENV</tt>.
27
+ # - If none of these are set, it will return <tt>development</tt>
28
+ #
29
+ # @return [String] The detected environment.
30
+ def environment
31
+ if defined?(Rails) && Rails.respond_to?(:env)
32
+ Rails.env.to_s
33
+ else
34
+ ENV['RAILS_ENV'] || ENV['RACK_ENV'] || ENV['ENV'] || 'development'
35
+ end
36
+ end
37
+
38
+ # Sets default values for sample rate and logger.
39
+ #
40
+ # - Default sample rate is set to the value in the STATSD_SAMPLE_RATE environment variable,
41
+ # or 1.0 otherwise. See {StatsD#default_sample_rate}
42
+ # - {StatsD#logger} is set to a logger that send output to stderr.
43
+ #
44
+ # If you are including this library inside a Rails environment, additional initialization will
45
+ # be done as part of the {StatsD::Instrument::Railtie}.
46
+ #
47
+ # @return [void]
48
+ def setup
49
+ StatsD.default_sample_rate = ENV.fetch('STATSD_SAMPLE_RATE', 1.0).to_f
50
+ StatsD.logger = Logger.new($stderr)
51
+ end
52
+ end
53
+
54
+ StatsD::Instrument::Environment.setup
@@ -0,0 +1,14 @@
1
+ module StatsD::Instrument::Helpers
2
+ def capture_statsd_calls(&block)
3
+ mock_backend = StatsD::Instrument::Backends::CaptureBackend.new
4
+ old_backend, StatsD.backend = StatsD.backend, mock_backend
5
+ block.call
6
+ mock_backend.collected_metrics
7
+ ensure
8
+ if old_backend.kind_of?(StatsD::Instrument::Backends::CaptureBackend)
9
+ old_backend.collected_metrics.concat(mock_backend.collected_metrics)
10
+ end
11
+
12
+ StatsD.backend = old_backend
13
+ end
14
+ end
@@ -0,0 +1,96 @@
1
+ require 'rspec/expectations'
2
+ require 'rspec/core/version'
3
+
4
+ module StatsD::Instrument::Matchers
5
+ CUSTOM_MATCHERS = {
6
+ increment: :c,
7
+ measure: :ms,
8
+ gauge: :g,
9
+ histogram: :h,
10
+ set: :s,
11
+ key_value: :kv
12
+ }
13
+
14
+ class Matcher
15
+ include RSpec::Matchers::Composable if RSpec::Core::Version::STRING.start_with?('3')
16
+ include StatsD::Instrument::Helpers
17
+
18
+ def initialize(metric_type, metric_name, options = {})
19
+ @metric_type = metric_type
20
+ @metric_name = metric_name
21
+ @options = options
22
+ end
23
+
24
+ def matches?(block)
25
+ begin
26
+ expect_statsd_call(@metric_type, @metric_name, @options, &block)
27
+ rescue RSpec::Expectations::ExpectationNotMetError => e
28
+ @message = e.message
29
+
30
+ false
31
+ end
32
+ end
33
+
34
+ def failure_message
35
+ @message
36
+ end
37
+
38
+ def failure_message_when_negated
39
+ "No StatsD calls for metric #{@metric_name} expected."
40
+ end
41
+
42
+ def supports_block_expectations?
43
+ true
44
+ end
45
+
46
+ private
47
+
48
+ def expect_statsd_call(metric_type, metric_name, options, &block)
49
+ metrics = capture_statsd_calls(&block)
50
+ metrics = metrics.select { |m| m.type == metric_type && m.name == metric_name }
51
+
52
+ raise RSpec::Expectations::ExpectationNotMetError, "No StatsD calls for metric #{metric_name} were made." if metrics.empty?
53
+ raise RSpec::Expectations::ExpectationNotMetError, "The numbers of StatsD calls for metric #{metric_name} was unexpected. Expected #{options[:times].inspect}, got #{metrics.length}" if options[:times] && options[:times] != metrics.length
54
+
55
+ [:sample_rate, :value, :tags].each do |expectation|
56
+ next unless options[expectation]
57
+
58
+ num_matches = metrics.count do |m|
59
+ matcher = RSpec::Matchers::BuiltIn::Match.new(options[expectation])
60
+ matcher.matches?(m.public_send(expectation))
61
+ end
62
+
63
+ found = options[:times] ? num_matches == options[:times] : num_matches > 0
64
+
65
+ if !found
66
+ message = metric_information(metric_name, options, metrics, expectation)
67
+ raise RSpec::Expectations::ExpectationNotMetError, message
68
+ end
69
+ end
70
+
71
+ true
72
+ end
73
+
74
+ def metric_information(metric_name, options, metrics, expectation)
75
+ message = "expected StatsD #{expectation.inspect} for metric '#{metric_name}' to be called"
76
+
77
+ message += "\n "
78
+ message += options[:times] ? "exactly #{options[:times]} times" : "at least once"
79
+ message += " with: #{options[expectation]}"
80
+
81
+ message += "\n captured metric values: #{metrics.map(&expectation).join(', ')}"
82
+
83
+ message
84
+ end
85
+ end
86
+
87
+ CUSTOM_MATCHERS.each do |method_name, metric_type|
88
+ klass = Class.new(Matcher)
89
+
90
+ define_method "trigger_statsd_#{method_name}" do |metric_name, options = {}|
91
+ klass.new(metric_type, metric_name, options)
92
+ end
93
+
94
+ StatsD::Instrument::Matchers.const_set(method_name.capitalize, klass)
95
+ end
96
+ end
@@ -0,0 +1,117 @@
1
+ # The Metric class represents a metric sample to be send by a backend.
2
+ #
3
+ # @!attribute type
4
+ # @return [Symbol] The metric type. Must be one of {StatsD::Instrument::Metric::TYPES}
5
+ # @!attribute name
6
+ # @return [String] The name of the metric. {StatsD#prefix} will automatically be applied
7
+ # to the metric in the constructor, unless the <tt>:no_prefix</tt> option is set.
8
+ # @!attribute value
9
+ # @see #default_value
10
+ # @return [Numeric, String] The value to collect for the metric. Depending on the metric
11
+ # type, <tt>value</tt> can be a string, integer, or float.
12
+ # @!attribute sample_rate
13
+ # The sample rate to use for the metric. How the sample rate is handled differs per backend.
14
+ # The UDP backend will actually sample metric submissions based on the sample rate, while
15
+ # the logger backend will just include the sample rate in its output for debugging purposes.
16
+ # @see StatsD#default_sample_rate
17
+ # @return [Float] The sample rate to use for this metric. This should be a value between
18
+ # 0 and 1. If not set, it will use the default sample rate set to {StatsD#default_sample_rate}.
19
+ # @!attribute tags
20
+ # The tags to associate with the metric.
21
+ # @note Only the Datadog implementation supports tags.
22
+ # @see .normalize_tags
23
+ # @return [Array<String>, Hash<String, String>, nil] the tags to associate with the metric.
24
+ # You can either specify the tags as an array of strings, or a Hash of key/value pairs.
25
+ #
26
+ # @see StatsD The StatsD module contains methods that generate metric instances.
27
+ # @see StatsD::Instrument::Backend A StatsD::Instrument::Backend is used to collect metrics.
28
+ #
29
+ class StatsD::Instrument::Metric
30
+
31
+ attr_accessor :type, :name, :value, :sample_rate, :tags
32
+
33
+ # Initializes a new metric instance.
34
+ # Normally, you don't want to call this method directly, but use one of the metric collection
35
+ # methods on the {StatsD} module.
36
+ #
37
+ # @option options [Symbol] :type The type of the metric.
38
+ # @option options [String] :name The name of the metric without prefix.
39
+ # @option options [Boolean] :no_prefix Set to <tt>true</tt> if you don't want to apply {StatsD#prefix}
40
+ # @option options [Numeric, String, nil] :value The value to collect for the metric. If set to
41
+ # <tt>nil>/tt>, {#default_value} will be used.
42
+ # @option options [Numeric, nil] :sample_rate The sample rate to use. If not set, it will use
43
+ # {StatsD#default_sample_rate}.
44
+ # @option options [Array<String>, Hash<String, String>, nil] :tags The tags to apply to this metric.
45
+ # See {.normalize_tags} for more information.
46
+ def initialize(options = {})
47
+ @type = options[:type] or raise ArgumentError, "Metric :type is required."
48
+ @name = options[:name] or raise ArgumentError, "Metric :name is required."
49
+ @name = normalize_name(@name)
50
+ @name = StatsD.prefix ? "#{StatsD.prefix}.#{@name}" : @name unless options[:no_prefix]
51
+
52
+ @value = options[:value] || default_value
53
+ @sample_rate = options[:sample_rate] || StatsD.default_sample_rate
54
+ @tags = StatsD::Instrument::Metric.normalize_tags(options[:tags])
55
+ end
56
+
57
+ # The default value for this metric, which will be used if it is not set.
58
+ #
59
+ # A default value is only defined for counter metrics (<tt>1</tt>). For all other
60
+ # metric types, this emthod will raise an <tt>ArgumentError</tt>.
61
+ #
62
+ # @return [Numeric, String] The default value for this metric.
63
+ # @raise ArgumentError if the metric type doesn't have a default value
64
+ def default_value
65
+ case type
66
+ when :c; 1
67
+ else raise ArgumentError, "A value is required for metric type #{type.inspect}."
68
+ end
69
+ end
70
+
71
+ # @private
72
+ # @return [String]
73
+ def to_s
74
+ str = "#{TYPES[type]} #{name}:#{value}"
75
+ str << " @#{sample_rate}" if sample_rate != 1.0
76
+ str << " " << tags.map { |t| "##{t}"}.join(' ') if tags
77
+ str
78
+ end
79
+
80
+ # @private
81
+ # @return [String]
82
+ def inspect
83
+ "#<StatsD::Instrument::Metric #{self.to_s}>"
84
+ end
85
+
86
+ # The metric types that are supported by this library. Note that every StatsD server
87
+ # implementation only supports a subset of them.
88
+ TYPES = {
89
+ c: 'increment',
90
+ ms: 'measure',
91
+ g: 'gauge',
92
+ h: 'histogram',
93
+ kv: 'key/value',
94
+ s: 'set',
95
+ }
96
+
97
+ # Strip metric names of special characters used by StatsD line protocol, replace with underscore
98
+ #
99
+ # @param name [String]
100
+ # @return [String]
101
+ def normalize_name(name)
102
+ name.tr(':|@'.freeze, '_')
103
+ end
104
+
105
+ # Utility function to convert tags to the canonical form.
106
+ #
107
+ # - Tags specified as key value pairs will be converted into an array
108
+ # - Tags are normalized to only use word characters and underscores.
109
+ #
110
+ # @param tags [Array<String>, Hash<String, String>, nil] Tags specified in any form.
111
+ # @return [Array<String>, nil] the list of tags in canonical form.
112
+ def self.normalize_tags(tags)
113
+ return unless tags
114
+ tags = tags.map { |k, v| k.to_s + ":".freeze + v.to_s } if tags.is_a?(Hash)
115
+ tags.map { |tag| tag.tr('|,'.freeze, ''.freeze) }
116
+ end
117
+ end
@@ -0,0 +1,67 @@
1
+ # @private
2
+ class StatsD::Instrument::MetricExpectation
3
+
4
+ attr_accessor :times, :type, :name, :value, :sample_rate, :tags
5
+ attr_reader :ignore_tags
6
+
7
+ def initialize(options = {})
8
+ @type = options[:type] or raise ArgumentError, "Metric :type is required."
9
+ @name = options[:name] or raise ArgumentError, "Metric :name is required."
10
+ @name = StatsD.prefix ? "#{StatsD.prefix}.#{@name}" : @name unless options[:no_prefix]
11
+ @tags = StatsD::Instrument::Metric.normalize_tags(options[:tags])
12
+ @times = options[:times] or raise ArgumentError, "Metric :times is required."
13
+ @sample_rate = options[:sample_rate]
14
+ @value = options[:value]
15
+ @ignore_tags = StatsD::Instrument::Metric.normalize_tags(options[:ignore_tags])
16
+ end
17
+
18
+ def matches(actual_metric)
19
+ return false if sample_rate && sample_rate != actual_metric.sample_rate
20
+ return false if value && value != actual_metric.value
21
+
22
+ if tags
23
+
24
+ expected_tags = Set.new(tags)
25
+ actual_tags = Set.new(actual_metric.tags)
26
+
27
+ if ignore_tags
28
+ ignored_tags = Set.new(ignore_tags) - expected_tags
29
+ actual_tags -= ignored_tags
30
+
31
+ if ignore_tags.is_a?(Array)
32
+ actual_tags.delete_if{ |key| ignore_tags.include?(key.split(":").first) }
33
+ end
34
+ end
35
+
36
+ return expected_tags.subset?(actual_tags)
37
+ end
38
+ true
39
+ end
40
+
41
+ def default_value
42
+ case type
43
+ when :c; 1
44
+ end
45
+ end
46
+
47
+ TYPES = {
48
+ c: 'increment',
49
+ ms: 'measure',
50
+ g: 'gauge',
51
+ h: 'histogram',
52
+ kv: 'key/value',
53
+ s: 'set',
54
+ }
55
+
56
+ def to_s
57
+ str = "#{TYPES[type]} #{name}:#{value}"
58
+ str << " @#{sample_rate}" if sample_rate != 1.0
59
+ str << " " << tags.map { |t| "##{t}"}.join(' ') if tags
60
+ str << " times:#{times}" if times > 1
61
+ str
62
+ end
63
+
64
+ def inspect
65
+ "#<StatsD::Instrument::MetricExpectation #{self.to_s}>"
66
+ end
67
+ end
@@ -0,0 +1,14 @@
1
+ # This Railtie runs some initializers that will set the logger to <tt>Rails#logger</tt>,
2
+ # and will initialize the {StatsD#backend} based on the Rails environment.
3
+ #
4
+ # @see StatsD::Instrument::Environment
5
+ class StatsD::Instrument::Railtie < Rails::Railtie
6
+
7
+ initializer 'statsd-instrument.use_rails_logger' do
8
+ ::StatsD.logger = Rails.logger
9
+ end
10
+
11
+ initializer 'statsd-instrument.setup_backend', after: 'statsd-instrument.use_rails_logger' do
12
+ ::StatsD.backend = ::StatsD::Instrument::Environment.default_backend
13
+ end
14
+ end
@@ -0,0 +1,5 @@
1
+ module StatsD
2
+ module Instrument
3
+ VERSION = "2.1.4"
4
+ end
5
+ end