tsdb_time_series 4.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,170 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ module Opower
4
+ module TimeSeries
5
+ # Represents a query that can be sent to an OpenTSDB instance through a [TSDBClient] object.
6
+ class Query
7
+ attr_accessor :metrics, :request, :response, :format
8
+
9
+ # Creates a new Query object.
10
+ #
11
+ # @param [Hash] config The configuration for this query.
12
+ # @option config [String] :format The format to return data with. Defaults to 'json'.
13
+ # @option config [String, Integer, DateTime] :start The start time. Required field.
14
+ # @option config [String, Integer, DateTime] :end The end time. Optional field.
15
+ # @option config [Hash] :m Array of metric hashes. This maps to what OpenTSDB expects as the metrics to query.
16
+ # * :aggregator [String] The aggregation type to utilize. Optional. Defaults to 'sum' if omitted.
17
+ # * :metric [String] The metric name. Required.
18
+ # * :tags [Hash] Hash consisting of Tag Key / Tag Value pairs. Optional.
19
+ # * :down_sample [Hash] to specify downsampling period and function
20
+ # * :period [String] The period of time to downsample one
21
+ # * :function [String] The function [min, max, sum, avg, dev]
22
+ # @option config [Boolean] padding If set to true, OpenTSDB (>= 2.0) will pad the start/end period.
23
+ #
24
+ # This object also supports all of the options available to the REST API for OpenTSDB.
25
+ # See http://opentsdb.net/http-api.html#/q_Parameters for more information.
26
+ #
27
+ # @return [Query] new Query object
28
+ def initialize(config = {})
29
+ @request = config
30
+ @format = config.delete(:format)
31
+
32
+ # Check that 'start' and 'm' parameters required by OpenTSDB are present
33
+ @requirements = [:start, :m]
34
+ validate_metrics
35
+
36
+ @metrics = @request.delete(:m)
37
+ check_metrics
38
+ convert_dates
39
+
40
+ # Create 'm' array - this is the
41
+ @request[:m] = @metrics.map(&MetricQuery.method(:new))
42
+ end
43
+
44
+ # Returns the current query as a URL to a PNG generated by OpenTSDB
45
+ #
46
+ # @return [String] url to gnuplot graph
47
+ def as_graph
48
+ GraphingRequest.new(@request).to_s
49
+ end
50
+
51
+ private
52
+
53
+ # Validates the query for the configured required fields. OpenTSDB requires that at the minimum the start time
54
+ # and a metric field (defined by 'm') to be present.
55
+ #
56
+ # @raise [ArgumentError] thrown if 'm' or 'start' are missing
57
+ def validate_metrics
58
+ keys = @request.keys
59
+ @requirements.each do |req|
60
+ next if keys.include?(req.to_sym) || keys.include?(req.to_s)
61
+ fail(ArgumentError, "#{req} is a required parameter.")
62
+ end
63
+ end
64
+
65
+ # Converts dates to a format that OpenTSDB understands.
66
+ def convert_dates
67
+ @request = Hash[@request.map do |key, value|
68
+ value = value.strftime('%Y/%m/%d-%H:%M:%S') if value.respond_to? :strftime
69
+ [key.to_s, value]
70
+ end]
71
+ end
72
+
73
+ # Checks the consistency of the metrics provided as defined in the user guide.
74
+ #
75
+ # @raise [ArgumentError] thrown if 'm' is in an unexpected state
76
+ def check_metrics
77
+ fail(ArgumentError, 'm parameter must be an array.') unless @metrics.is_a? Array
78
+ fail(ArgumentError, 'm parameter must not be empty.') unless @metrics.length > 0
79
+
80
+ @metrics.each do |metric|
81
+ fail(ArgumentError, "Expected a Hash - got a #{metric.class}: '#{metric}'") unless metric.is_a? Hash
82
+ fail(ArgumentError, 'Metric label must be present for query to run.') unless metric.key?(:metric)
83
+ end
84
+ end
85
+
86
+ # Wrapper class for each query made against a separate metric.
87
+ class MetricQuery
88
+ # Initializes a wrapper object that represent a single metric that will be queried against in OpenTSDB.
89
+ #
90
+ # @param [Hash] metric a Hash representing a query against OpenTSDB
91
+ def initialize(metric)
92
+ @metric = metric.fetch(:metric)
93
+ @tags = metric.fetch(:tags, {})
94
+ @aggregator = metric.fetch(:aggregator, 'sum')
95
+ @rate = metric.fetch(:rate, false)
96
+
97
+ initialize_downsample(metric)
98
+ end
99
+
100
+ # Builds the query string representation of this MetricQuery object.
101
+ #
102
+ # @return [String] the URL query string that will be used for this metric query.
103
+ def to_s
104
+ str = "#{@aggregator}:"
105
+ str << "#{@period}-#{@function}:" if @downsample
106
+ str << 'rate:' if @rate
107
+ str << "#{@metric}{#{build_tags}}"
108
+ str
109
+ end
110
+
111
+ private
112
+
113
+ # Initializes the down-sample setting on the metric
114
+ #
115
+ # @param [Hash] metric the metric hash
116
+ def initialize_downsample(metric)
117
+ @downsample = false
118
+ return unless metric.key?(:downsample)
119
+
120
+ down_sample = metric[:downsample]
121
+ @period = down_sample[:period]
122
+ @function = down_sample[:function]
123
+ @downsample = true
124
+ end
125
+
126
+ # Builds the string representation of the tags for the metric
127
+ #
128
+ # @return query string for tag metrics
129
+ def build_tags
130
+ @tags.map { |key, value| "#{key}=#{value}" }.join(',')
131
+ end
132
+ end
133
+
134
+ # Generates a graphing request URL from a properly configured Query object.
135
+ class GraphingRequest
136
+ # Initializes the graphing request wrapper using the data found in the Query object.
137
+ #
138
+ # @param [Hash] request query configuration
139
+ def initialize(request)
140
+ @parameters = request
141
+ @request = []
142
+ build_request
143
+ end
144
+
145
+ # Returns the URL for the specified query and its requested data.
146
+ #
147
+ # @return [String] URI address for the specified query
148
+ def to_s
149
+ URI.encode(@request.join('&'))
150
+ end
151
+
152
+ private
153
+
154
+ # Builds the query string for the OpenTSDB REST API.
155
+ # This method smells of :reek:NestedIterators
156
+ #
157
+ # @return [String] the GET query string for this object.
158
+ def build_request
159
+ @parameters.each_pair do |key, value|
160
+ if value.respond_to? :each
161
+ value.each { |array_element| @request << "#{key}=#{array_element.to_s.strip}" }
162
+ else
163
+ @request << "#{key}=#{value.to_s.strip}"
164
+ end
165
+ end
166
+ end
167
+ end
168
+ end
169
+ end
170
+ end
@@ -0,0 +1,40 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ module Opower
4
+ module TimeSeries
5
+ # Wraps the OpenTSDB result with response codes and result counts
6
+ class Result
7
+ attr_reader :status, :length, :results, :error_message
8
+
9
+ # Takes the Excon response from OpenTSDB and parses it into the desired format.
10
+ #
11
+ # @param [Response] response The Excon response object
12
+ def initialize(response)
13
+ @status = response.status
14
+ @length = 0
15
+ data = response.body
16
+
17
+ parse_results(data)
18
+ end
19
+
20
+ # Checks if the status code is not a 2XX HTTP response code.
21
+ #
22
+ # @return [Boolean] true if an error occurred
23
+ def errors?
24
+ @status.to_s !~ /^2/
25
+ end
26
+
27
+ private
28
+
29
+ # Parses the results from OpenTSDB
30
+ #
31
+ # @param [String] data HTTP Response Body
32
+ def parse_results(data)
33
+ @results = JSON.parse(data)
34
+ @length = @results.length
35
+
36
+ @error_message = @results['error']['message'] if errors? && @length > 0
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,103 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ require 'dentaku'
4
+
5
+ module Opower
6
+ module TimeSeries
7
+ # Provides support for synthetic metrics
8
+ class SyntheticResult
9
+ attr_reader :results, :data
10
+
11
+ # Initializes a synthetic results wrapper and runs the calculations on the results data.
12
+ #
13
+ # @param name [String] - alias for this synthetic series
14
+ # @param formula [String] - the formula used to calculate data together
15
+ # @param data [Hash] - a hash containing key mappings to results to be used in the formula
16
+ def initialize(name, formula, data)
17
+ @name = name
18
+ @formula = formula
19
+ @data = data
20
+ @results = {}
21
+ @calculator = TimeSeriesCalculator.new
22
+
23
+ calculate
24
+ end
25
+
26
+ # Calculates the result of the formula set for this synthetic result. Currently, the timestamps
27
+ # of each of the queries must match in order for a calculation to be performed.
28
+ def calculate
29
+ formula_map = FormulaMap.new(@data)
30
+
31
+ formula_map.parameters.each do |ts, parameter|
32
+ @results[ts] = @calculator.evaluate(@formula, parameter)
33
+ end
34
+ end
35
+
36
+ # Gives the total results size after calculating synthetic metrics.
37
+ #
38
+ # @return [Integer] the number of data-points
39
+ def length
40
+ @results.keys.length
41
+ end
42
+
43
+ # Subclass of Dentaku's calculator - adds math functions by default
44
+ class TimeSeriesCalculator < Dentaku::Calculator
45
+ def initialize
46
+ super
47
+ initialize_math_functions
48
+ end
49
+
50
+ # Initializes math functions provided by Ruby and places them into the calculator as functions.
51
+ # NOTE: You must wrap nested mathematical expressions in formulas or Dentaku will attempt to pass them
52
+ # as separate arguments into the lambda below!
53
+ # This method smells of :reek:NestedIterators
54
+ #
55
+ # For example:
56
+ # Assume x = 1, y = 2
57
+ # cos(x + y) is translated into cos(1, 'add', 2) - this calls Math.cos(1, 'add', 2)
58
+ # cos((x + y)) is translated into cos(3) - this correctly calls Math.cos(3)
59
+ def initialize_math_functions
60
+ Math.methods(false).each do |method|
61
+ math_method = Math.method(method)
62
+ add_function(
63
+ name: method,
64
+ type: :numeric,
65
+ signature: [:numeric],
66
+ body: ->(*args) { math_method.call(*args) }
67
+ )
68
+ end
69
+ end
70
+ end
71
+
72
+ # Merges all matching data point timestamps together and assigns their values to formula keys
73
+ class FormulaMap
74
+ attr_reader :parameters
75
+
76
+ # Initializes the formula map using the set of results received from OpenTSDB.
77
+ #
78
+ # @param [Hash] data A hash mapping formula parameters to OpenTSDB results
79
+ def initialize(data)
80
+ @data = data
81
+ @parameters = {}
82
+
83
+ @data[@data.keys.sample].each_key do |ts|
84
+ build_hash(ts)
85
+ end
86
+ end
87
+
88
+ # Checks each of the keys in the data provided to see if they contain a data-point at the
89
+ # specified timestamp. Does nothing if any of the keys is missing a data-point.
90
+ #
91
+ # @param [String] ts the OpenTSDB timestamp to check (yes, it's a String from OpenTSDB)
92
+ def build_hash(ts)
93
+ result = @data.map { |key, dps| { key => dps[ts] } if dps.key?(ts) }
94
+ return if result.include?(nil)
95
+
96
+ @parameters[ts] = result.reduce do |formula_map, parameter|
97
+ formula_map.merge(parameter)
98
+ end
99
+ end
100
+ end
101
+ end
102
+ end
103
+ end
@@ -0,0 +1,169 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ require 'rubygems'
4
+ require 'excon'
5
+ require 'json'
6
+ require 'logger'
7
+
8
+ module Opower
9
+ module TimeSeries
10
+ # Ruby client object to interface with an OpenTSDB instance.
11
+ class TSClient
12
+ attr_accessor :host, :port, :client, :config, :connection, :connection_settings
13
+
14
+ # Creates a connection to a specified OpenTSDB instance
15
+ #
16
+ # @param [String] host The hostname/IP to connect to. Defaults to 'localhost'.
17
+ # @param [Integer] port The port to connect to. Defaults to 4242.
18
+ # @param [String] protocol The protocol to use. Defaults to 'http'.
19
+ #
20
+ # @return [TSClient] Client connection to OpenTSDB.
21
+ def initialize(host = '127.0.0.1', port = 4242, protocol = 'http')
22
+ @host = host
23
+ @port = port
24
+
25
+ @client = "#{protocol}://#{host}:#{port}/"
26
+ @connection = Excon.new(@client, persistent: true, idempotent: true, tcp_nodelay: true)
27
+ @connection_settings = @connection.data
28
+ configure
29
+ end
30
+
31
+ # Configures client-specific options
32
+ #
33
+ # @param [Hash] cfg The configuration options to set.
34
+ # @option cfg [Boolean] :dry_run When set to true, the client does not actually read/write to OpenTSDB.
35
+ # @option cfg [String] :version The version of OpenTSDB to run against. Defaults to 2.0.
36
+ def configure(cfg = {})
37
+ @config = { dry_run: false, version: '2.0' }
38
+ @valid_config_keys = @config.keys
39
+
40
+ cfg.each do |key, value|
41
+ key = key.to_sym
42
+ @config[key] = value if @valid_config_keys.include?(key)
43
+ end
44
+ end
45
+
46
+ # Basic check to see if the OpenTSDB is reachable
47
+ #
48
+ # @return [Boolean] true if call against api/version resolves
49
+ def valid?
50
+ @connection.get(path: 'api/version')
51
+ true
52
+ rescue Excon::Errors::SocketError, Excon::Errors::Timeout
53
+ false
54
+ end
55
+
56
+ # Returns suggestions for metric or tag names
57
+ #
58
+ # @param [String] query The string to search for
59
+ # @param [String] type The type to search for: 'metrics', 'tagk', 'tagv'
60
+ #
61
+ # @return [Array] an array of possible values based on the query/type
62
+ def suggest(query, type = 'metrics', max = 25)
63
+ return suggest_uri(query, type, max) if @config[:dry_run]
64
+ JSON.parse(@connection.get(path: 'api/suggest', query: { type: type, q: query, max: max }).body)
65
+ end
66
+
67
+ # Returns the full URI for the suggest query in the context of this client.
68
+ #
69
+ # @param [String] query The string to search for
70
+ # @param [String] type The type to search for: 'metrics', 'tagk', 'tagv'
71
+ # @return [String] the URI
72
+ def suggest_uri(query, type = 'metrics', max = 25)
73
+ @client + "api/suggest?type=#{type}&q=#{query}&max=#{max}"
74
+ end
75
+
76
+ # Writes the specified Metric object to OpenTSDB.
77
+ #
78
+ # @param [Metric] metric The metric to write to OpenTSDB
79
+ def write(metric)
80
+ cmd = "echo \"put #{metric}\" | nc -w 30 #{@host} #{@port}"
81
+
82
+ if @config[:dry_run]
83
+ cmd
84
+ else
85
+ # Write into the db
86
+ ret = system(cmd)
87
+
88
+ # Command failed to run
89
+ fail(IOError, "Failed to insert metric #{metric.name} with value of #{metric.value} into OpenTSDB.") unless ret
90
+ end
91
+ end
92
+
93
+ # Runs the specified query against OpenTSDB. If config[:dry-run] is set to true or PNG format requested,
94
+ # it will only return the URL for the query. Otherwise it will return a Result object.
95
+ #
96
+ # @param [Query] query The query object to execute with.
97
+ # @return [Result || String] the results of the query
98
+ def run_query(query)
99
+ return query_uri(query) if @config[:dry_run] || query.format == :png
100
+ Result.new(@connection.get(path: 'api/query', query: query.request))
101
+ end
102
+
103
+ # Returns the full URI for the query in the context of this client.
104
+ #
105
+ # @param [Query] query The query object
106
+ # @return [String] the URI
107
+ def query_uri(query)
108
+ @client + 'api/query?' + query.as_graph
109
+ end
110
+
111
+ # Runs a synthetic query using queries against OpenTSDB. It expects a formula and a matching Hash which maps
112
+ # parameters in the formula to time_series' query objects.
113
+ #
114
+ # @param name [String] the alias for this synthetic series
115
+ # @param formula [String] the Dentaku calculator formula to use
116
+ # @param query_hash [Hash] a Hash containing Query objects that map to corresponding parameters in the formula
117
+ # @return [SyntheticResult] the calculated result of the formula
118
+ def run_synthetic_query(name, formula, query_hash)
119
+ results_hash = query_hash.map { |key, query| { key => run_query(query).results[0].fetch('dps') } }
120
+ results_hash = results_hash.reduce do |results, result|
121
+ results.merge(result)
122
+ end
123
+
124
+ SyntheticResult.new(name, formula, results_hash)
125
+ end
126
+
127
+ # Runs the specified queries against OpenTSDB in a HTTP pipelined connection.
128
+ #
129
+ # @param [Array] queries An array of queries to run against OpenTSDB.
130
+ # @return [Array] a matching array of results for each query
131
+ def run_queries(queries)
132
+ # requests cannot be idempotent when pipelined, so we temporarily disable it
133
+ @connection_settings[:idempotent] = false
134
+
135
+ results = run_pipelined_request(queries)
136
+
137
+ @connection_settings[:idempotent] = true
138
+ results
139
+ end
140
+
141
+ private
142
+
143
+ # Runs a series of queries in a pipelined, persistent HTTP request against OpenTSDB.
144
+ #
145
+ # @param [Array] queries Array of Query objects to run against OpenTSDB
146
+ # @return [Array] corresponding Array of Result objects
147
+ def run_pipelined_request(queries)
148
+ wrapper = PipelineWrapper.new(@config, queries)
149
+ responses = @connection.requests(wrapper.requests)
150
+ responses.map { |response| Result.new(response) }
151
+ end
152
+
153
+ # Wraps pipelined requests and creates their individual HTTP requests against OpenTSDB
154
+ class PipelineWrapper
155
+ attr_reader :requests
156
+
157
+ # Initializes the pipeline wrapper and sets up the Excon requests based on the queries.
158
+ #
159
+ # @param [Hash] config the client configuration
160
+ # @param [Array] queries the Array of Query objects to execute
161
+ def initialize(config, queries)
162
+ @config = config
163
+ @queries = queries
164
+ @requests = @queries.map { |query| { method: :get, path: 'api/query', query: query.request } }
165
+ end
166
+ end
167
+ end
168
+ end
169
+ end