tsdb_time_series 4.1.2

data/lib/time_series/query.rb

@@ -0,0 +1,170 @@
+# -*- encoding: utf-8 -*-
+
+module Opower
+  module TimeSeries
+    # Represents a query that can be sent to an OpenTSDB instance through a [TSDBClient] object.
+    class Query
+      attr_accessor :metrics, :request, :response, :format
+
+      # Creates a new Query object.
+      #
+      # @param [Hash] config The configuration for this query.
+      # @option config [String] :format The format to return data with. Defaults to 'json'.
+      # @option config [String, Integer, DateTime] :start The start time. Required field.
+      # @option config [String, Integer, DateTime] :end The end time. Optional field.
+      # @option config [Hash] :m Array of metric hashes. This maps to what OpenTSDB expects as the metrics to query.
+      #  * :aggregator [String] The aggregation type to utilize. Optional. Defaults to 'sum' if omitted.
+      #  * :metric [String] The metric name. Required.
+      #  * :tags [Hash] Hash consisting of Tag Key / Tag Value pairs. Optional.
+      #  * :down_sample [Hash] to specify downsampling period and function
+      #    * :period [String] The period of time to downsample one
+      #    * :function [String] The function [min, max, sum, avg, dev]
+      # @option config [Boolean] padding If set to true, OpenTSDB (>= 2.0) will pad the start/end period.
+      #
+      # This object also supports all of the options available to the REST API for OpenTSDB.
+      # See http://opentsdb.net/http-api.html#/q_Parameters for more information.
+      #
+      # @return [Query] new Query object
+      def initialize(config = {})
+        @request = config
+        @format = config.delete(:format)
+
+        # Check that 'start' and 'm' parameters required by OpenTSDB are present
+        @requirements = [:start, :m]
+        validate_metrics
+
+        @metrics = @request.delete(:m)
+        check_metrics
+        convert_dates
+
+        # Create 'm' array - this is the
+        @request[:m] = @metrics.map(&MetricQuery.method(:new))
+      end
+
+      # Returns the current query as a URL to a PNG generated by OpenTSDB
+      #
+      # @return [String] url to gnuplot graph
+      def as_graph
+        GraphingRequest.new(@request).to_s
+      end
+
+      private
+
+      # Validates the query for the configured required fields. OpenTSDB requires that at the minimum the start time
+      # and a metric field (defined by 'm') to be present.
+      #
+      # @raise [ArgumentError] thrown if 'm' or 'start' are missing
+      def validate_metrics
+        keys = @request.keys
+        @requirements.each do |req|
+          next if keys.include?(req.to_sym) || keys.include?(req.to_s)
+          fail(ArgumentError, "#{req} is a required parameter.")
+        end
+      end
+
+      # Converts dates to a format that OpenTSDB understands.
+      def convert_dates
+        @request = Hash[@request.map do |key, value|
+          value = value.strftime('%Y/%m/%d-%H:%M:%S') if value.respond_to? :strftime
+          [key.to_s, value]
+        end]
+      end
+
+      # Checks the consistency of the metrics provided as defined in the user guide.
+      #
+      # @raise [ArgumentError] thrown if 'm' is in an unexpected state
+      def check_metrics
+        fail(ArgumentError, 'm parameter must be an array.') unless @metrics.is_a? Array
+        fail(ArgumentError, 'm parameter must not be empty.') unless @metrics.length > 0
+
+        @metrics.each do |metric|
+          fail(ArgumentError, "Expected a Hash - got a #{metric.class}: '#{metric}'") unless metric.is_a? Hash
+          fail(ArgumentError, 'Metric label must be present for query to run.') unless metric.key?(:metric)
+        end
+      end
+
+      # Wrapper class for each query made against a separate metric.
+      class MetricQuery
+        # Initializes a wrapper object that represent a single metric that will be queried against in OpenTSDB.
+        #
+        # @param [Hash] metric a Hash representing a query against OpenTSDB
+        def initialize(metric)
+          @metric = metric.fetch(:metric)
+          @tags = metric.fetch(:tags, {})
+          @aggregator = metric.fetch(:aggregator, 'sum')
+          @rate = metric.fetch(:rate, false)
+
+          initialize_downsample(metric)
+        end
+
+        # Builds the query string representation of this MetricQuery object.
+        #
+        # @return [String] the URL query string that will be used for this metric query.
+        def to_s
+          str = "#{@aggregator}:"
+          str << "#{@period}-#{@function}:" if @downsample
+          str << 'rate:' if @rate
+          str << "#{@metric}{#{build_tags}}"
+          str
+        end
+
+        private
+
+        # Initializes the down-sample setting on the metric
+        #
+        # @param [Hash] metric the metric hash
+        def initialize_downsample(metric)
+          @downsample = false
+          return unless metric.key?(:downsample)
+
+          down_sample = metric[:downsample]
+          @period = down_sample[:period]
+          @function = down_sample[:function]
+          @downsample = true
+        end
+
+        # Builds the string representation of the tags for the metric
+        #
+        # @return query string for tag metrics
+        def build_tags
+          @tags.map { |key, value| "#{key}=#{value}" }.join(',')
+        end
+      end
+
+      # Generates a graphing request URL from a properly configured Query object.
+      class GraphingRequest
+        # Initializes the graphing request wrapper using the data found in the Query object.
+        #
+        # @param [Hash] request query configuration
+        def initialize(request)
+          @parameters = request
+          @request = []
+          build_request
+        end
+
+        # Returns the URL for the specified query and its requested data.
+        #
+        # @return [String] URI address for the specified query
+        def to_s
+          URI.encode(@request.join('&'))
+        end
+
+        private
+
+        # Builds the query string for the OpenTSDB REST API.
+        # This method smells of :reek:NestedIterators
+        #
+        # @return [String] the GET query string for this object.
+        def build_request
+          @parameters.each_pair do |key, value|
+            if value.respond_to? :each
+              value.each { |array_element| @request << "#{key}=#{array_element.to_s.strip}" }
+            else
+              @request << "#{key}=#{value.to_s.strip}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/time_series/results/result.rb

@@ -0,0 +1,40 @@
+# -*- encoding: utf-8 -*-
+
+module Opower
+  module TimeSeries
+    # Wraps the OpenTSDB result with response codes and result counts
+    class Result
+      attr_reader :status, :length, :results, :error_message
+
+      # Takes the Excon response from OpenTSDB and parses it into the desired format.
+      #
+      # @param [Response] response The Excon response object
+      def initialize(response)
+        @status = response.status
+        @length = 0
+        data = response.body
+
+        parse_results(data)
+      end
+
+      # Checks if the status code is not a 2XX HTTP response code.
+      #
+      # @return [Boolean] true if an error occurred
+      def errors?
+        @status.to_s !~ /^2/
+      end
+
+      private
+
+      # Parses the results from OpenTSDB
+      #
+      # @param [String] data HTTP Response Body
+      def parse_results(data)
+        @results = JSON.parse(data)
+        @length = @results.length
+
+        @error_message = @results['error']['message'] if errors? && @length > 0
+      end
+    end
+  end
+end

data/lib/time_series/results/synthetic_result.rb

@@ -0,0 +1,103 @@
+# -*- encoding: utf-8 -*-
+
+require 'dentaku'
+
+module Opower
+  module TimeSeries
+    # Provides support for synthetic metrics
+    class SyntheticResult
+      attr_reader :results, :data
+
+      # Initializes a synthetic results wrapper and runs the calculations on the results data.
+      #
+      # @param name [String] - alias for this synthetic series
+      # @param formula [String] - the formula used to calculate data together
+      # @param data [Hash] - a hash containing key mappings to results to be used in the formula
+      def initialize(name, formula, data)
+        @name = name
+        @formula = formula
+        @data = data
+        @results = {}
+        @calculator = TimeSeriesCalculator.new
+
+        calculate
+      end
+
+      # Calculates the result of the formula set for this synthetic result. Currently, the timestamps
+      # of each of the queries must match in order for a calculation to be performed.
+      def calculate
+        formula_map = FormulaMap.new(@data)
+
+        formula_map.parameters.each do |ts, parameter|
+          @results[ts] = @calculator.evaluate(@formula, parameter)
+        end
+      end
+
+      # Gives the total results size after calculating synthetic metrics.
+      #
+      # @return [Integer] the number of data-points
+      def length
+        @results.keys.length
+      end
+
+      # Subclass of Dentaku's calculator - adds math functions by default
+      class TimeSeriesCalculator < Dentaku::Calculator
+        def initialize
+          super
+          initialize_math_functions
+        end
+
+        # Initializes math functions provided by Ruby and places them into the calculator as functions.
+        # NOTE: You must wrap nested mathematical expressions in formulas or Dentaku will attempt to pass them
+        # as separate arguments into the lambda below!
+        # This method smells of :reek:NestedIterators
+        #
+        # For example:
+        # Assume x = 1, y = 2
+        # cos(x + y) is translated into cos(1, 'add', 2) - this calls Math.cos(1, 'add', 2)
+        # cos((x + y)) is translated into cos(3) - this correctly calls Math.cos(3)
+        def initialize_math_functions
+          Math.methods(false).each do |method|
+            math_method = Math.method(method)
+            add_function(
+                name: method,
+                type: :numeric,
+                signature: [:numeric],
+                body: ->(*args) { math_method.call(*args) }
+            )
+          end
+        end
+      end
+
+      # Merges all matching data point timestamps together and assigns their values to formula keys
+      class FormulaMap
+        attr_reader :parameters
+
+        # Initializes the formula map using the set of results received from OpenTSDB.
+        #
+        # @param [Hash] data A hash mapping formula parameters to OpenTSDB results
+        def initialize(data)
+          @data = data
+          @parameters = {}
+
+          @data[@data.keys.sample].each_key do |ts|
+            build_hash(ts)
+          end
+        end
+
+        # Checks each of the keys in the data provided to see if they contain a data-point at the
+        # specified timestamp. Does nothing if any of the keys is missing a data-point.
+        #
+        # @param [String] ts the OpenTSDB timestamp to check (yes, it's a String from OpenTSDB)
+        def build_hash(ts)
+          result = @data.map { |key, dps| { key => dps[ts] } if dps.key?(ts) }
+          return if result.include?(nil)
+
+          @parameters[ts] = result.reduce do |formula_map, parameter|
+            formula_map.merge(parameter)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/time_series/ts_client.rb

@@ -0,0 +1,169 @@
+# -*- encoding: utf-8 -*-
+
+require 'rubygems'
+require 'excon'
+require 'json'
+require 'logger'
+
+module Opower
+  module TimeSeries
+    # Ruby client object to interface with an OpenTSDB instance.
+    class TSClient
+      attr_accessor :host, :port, :client, :config, :connection, :connection_settings
+
+      # Creates a connection to a specified OpenTSDB instance
+      #
+      # @param [String] host The hostname/IP to connect to. Defaults to 'localhost'.
+      # @param [Integer] port The port to connect to. Defaults to 4242.
+      # @param [String] protocol The protocol to use. Defaults to 'http'.
+      #
+      # @return [TSClient] Client connection to OpenTSDB.
+      def initialize(host = '127.0.0.1', port = 4242, protocol = 'http')
+        @host = host
+        @port = port
+
+        @client = "#{protocol}://#{host}:#{port}/"
+        @connection = Excon.new(@client, persistent: true, idempotent: true, tcp_nodelay: true)
+        @connection_settings = @connection.data
+        configure
+      end
+
+      # Configures client-specific options
+      #
+      # @param [Hash] cfg The configuration options to set.
+      # @option cfg [Boolean] :dry_run When set to true, the client does not actually read/write to OpenTSDB.
+      # @option cfg [String] :version The version of OpenTSDB to run against. Defaults to 2.0.
+      def configure(cfg = {})
+        @config = { dry_run: false, version: '2.0' }
+        @valid_config_keys = @config.keys
+
+        cfg.each do |key, value|
+          key = key.to_sym
+          @config[key] = value if @valid_config_keys.include?(key)
+        end
+      end
+
+      # Basic check to see if the OpenTSDB is reachable
+      #
+      # @return [Boolean] true if call against api/version resolves
+      def valid?
+        @connection.get(path: 'api/version')
+        true
+      rescue Excon::Errors::SocketError, Excon::Errors::Timeout
+        false
+      end
+
+      # Returns suggestions for metric or tag names
+      #
+      # @param [String] query The string to search for
+      # @param [String] type The type to search for: 'metrics', 'tagk', 'tagv'
+      #
+      # @return [Array] an array of possible values based on the query/type
+      def suggest(query, type = 'metrics', max = 25)
+        return suggest_uri(query, type, max) if @config[:dry_run]
+        JSON.parse(@connection.get(path: 'api/suggest', query: { type: type, q: query, max: max }).body)
+      end
+
+      # Returns the full URI for the suggest query in the context of this client.
+      #
+      # @param [String] query The string to search for
+      # @param [String] type The type to search for: 'metrics', 'tagk', 'tagv'
+      # @return [String] the URI
+      def suggest_uri(query, type = 'metrics', max = 25)
+        @client + "api/suggest?type=#{type}&q=#{query}&max=#{max}"
+      end
+
+      # Writes the specified Metric object to OpenTSDB.
+      #
+      # @param [Metric] metric The metric to write to OpenTSDB
+      def write(metric)
+        cmd = "echo \"put #{metric}\" | nc -w 30 #{@host} #{@port}"
+
+        if @config[:dry_run]
+          cmd
+        else
+          # Write into the db
+          ret = system(cmd)
+
+          # Command failed to run
+          fail(IOError, "Failed to insert metric #{metric.name} with value of #{metric.value} into OpenTSDB.") unless ret
+        end
+      end
+
+      # Runs the specified query against OpenTSDB. If config[:dry-run] is set to true or PNG format requested,
+      # it will only return the URL for the query. Otherwise it will return a Result object.
+      #
+      # @param [Query] query The query object to execute with.
+      # @return [Result || String] the results of the query
+      def run_query(query)
+        return query_uri(query) if @config[:dry_run] || query.format == :png
+        Result.new(@connection.get(path: 'api/query', query: query.request))
+      end
+
+      # Returns the full URI for the query in the context of this client.
+      #
+      # @param [Query] query The query object
+      # @return [String] the URI
+      def query_uri(query)
+        @client + 'api/query?' + query.as_graph
+      end
+
+      # Runs a synthetic query using queries against OpenTSDB. It expects a formula and a matching Hash which maps
+      # parameters in the formula to time_series' query objects.
+      #
+      # @param name [String] the alias for this synthetic series
+      # @param formula [String] the Dentaku calculator formula to use
+      # @param query_hash [Hash] a Hash containing Query objects that map to corresponding parameters in the formula
+      # @return [SyntheticResult] the calculated result of the formula
+      def run_synthetic_query(name, formula, query_hash)
+        results_hash = query_hash.map { |key, query| { key => run_query(query).results[0].fetch('dps') } }
+        results_hash = results_hash.reduce do |results, result|
+          results.merge(result)
+        end
+
+        SyntheticResult.new(name, formula, results_hash)
+      end
+
+      # Runs the specified queries against OpenTSDB in a HTTP pipelined connection.
+      #
+      # @param [Array] queries An array of queries to run against OpenTSDB.
+      # @return [Array] a matching array of results for each query
+      def run_queries(queries)
+        # requests cannot be idempotent when pipelined, so we temporarily disable it
+        @connection_settings[:idempotent] = false
+
+        results = run_pipelined_request(queries)
+
+        @connection_settings[:idempotent] = true
+        results
+      end
+
+      private
+
+      # Runs a series of queries in a pipelined, persistent HTTP request against OpenTSDB.
+      #
+      # @param [Array] queries Array of Query objects to run against OpenTSDB
+      # @return [Array] corresponding Array of Result objects
+      def run_pipelined_request(queries)
+        wrapper = PipelineWrapper.new(@config, queries)
+        responses = @connection.requests(wrapper.requests)
+        responses.map { |response| Result.new(response) }
+      end
+
+      # Wraps pipelined requests and creates their individual HTTP requests against OpenTSDB
+      class PipelineWrapper
+        attr_reader :requests
+
+        # Initializes the pipeline wrapper and sets up the Excon requests based on the queries.
+        #
+        # @param [Hash] config the client configuration
+        # @param [Array] queries the Array of Query objects to execute
+        def initialize(config, queries)
+          @config = config
+          @queries = queries
+          @requests = @queries.map { |query| { method: :get, path: 'api/query', query: query.request } }
+        end
+      end
+    end
+  end
+end