ruby-grafana-reporter 0.4.1 → 0.4.5

data/lib/grafana/variable.rb

@@ -33,6 +33,7 @@ module Grafana
       if config_or_value.is_a? Hash
         @config = config_or_value
         @name = @config['name']
+        # TODO: if a variable uses type 'query' which is never updated, the selected values are stored in 'options'
         unless @config['current'].nil?
           @raw_value = @config['current']['value']
           @text = @config['current']['text']
@@ -62,93 +63,98 @@ module Grafana
     def value_formatted(format = '')
       value = @raw_value
 
-      # handle value 'All' properly
-      # TODO: fix check for selection of All properly
-      if (value == 'All') || (@text == 'All')
+      # if 'All' is selected for this template variable, capture all values properly
+      # (from grafana config or query) and format the results afterwards accordingly
+      if value == '$__all'
         if !@config['options'].empty?
+          # this query contains predefined values, so capture them and format the values accordingly
+          # this happens either if type='custom' or a query, which is never updated
           value = @config['options'].map { |item| item['value'] }
-        elsif !@config['query'].empty?
-          # TODO: replace variables in this query, too
+
+        elsif @config['type'] == 'query' && !@config['query'].empty?
+          # TODO: replace variables in query, execute it and evaluate the results as if they were normally selected
+          # "multiFormat": contains variable replacement in "query", e.g. 'regex values' or 'glob'
+          # "datasource": contains name of datasource
           return @config['query']
-          # TODO: handle 'All' value properly for query attributes
+
         else
-          # TODO: how to handle All selection properly at this point?
+          # TODO: add support for variable type: 'datasource' and 'adhoc'
         end
       end
 
       case format
       when 'csv'
-        return value.join(',').to_s if multi?
+        return value.join(',').to_s if multi? && value.is_a?(Array)
 
         value.to_s
 
       when 'distributed'
-        return value.join(",#{name}=") if multi?
+        return value.join(",#{name}=") if multi? && value.is_a?(Array)
 
         value
       when 'doublequote'
-        if multi?
+        if multi? && value.is_a?(Array)
           value = value.map { |item| "\"#{item.gsub(/\\/, '\\\\').gsub(/"/, '\\"')}\"" }
           return value.join(',')
         end
         "\"#{value.gsub(/"/, '\\"')}\""
 
       when 'json'
-        if multi?
+        if multi? && value.is_a?(Array)
           value = value.map { |item| "\"#{item.gsub(/["\\]/, '\\\\\0')}\"" }
           return "[#{value.join(',')}]"
         end
         "\"#{value.gsub(/"/, '\\"')}\""
 
       when 'percentencode'
-        value = "{#{value.join(',')}}" if multi?
+        value = "{#{value.join(',')}}" if multi? && value.is_a?(Array)
         ERB::Util.url_encode(value)
 
       when 'pipe'
-        return value.join('|') if multi?
+        return value.join('|') if multi? && value.is_a?(Array)
 
         value
 
       when 'raw'
-        return "{#{value.join(',')}}" if multi?
+        return "{#{value.join(',')}}" if multi? && value.is_a?(Array)
 
         value
 
       when 'regex'
-        if multi?
+        if multi? && value.is_a?(Array)
           value = value.map { |item| item.gsub(%r{[/$.|\\]}, '\\\\\0') }
           return "(#{value.join('|')})"
         end
         value.gsub(%r{[/$.|\\]}, '\\\\\0')
 
       when 'singlequote'
-        if multi?
+        if multi? && value.is_a?(Array)
           value = value.map { |item| "'#{item.gsub(/'/, '\\\\\0')}'" }
           return value.join(',')
         end
         "'#{value.gsub(/'/, '\\\\\0')}'"
 
       when 'sqlstring'
-        if multi?
+        if multi? && value.is_a?(Array)
           value = value.map { |item| "'#{item.gsub(/'/, "''")}'" }
           return value.join(',')
         end
         "'#{value.gsub(/'/, "''")}'"
 
       when 'lucene'
-        if multi?
+        if multi? && value.is_a?(Array)
           value = value.map { |item| "\"#{item.gsub(%r{[" |=/\\]}, '\\\\\0')}\"" }
           return "(#{value.join(' OR ')})"
         end
         value.gsub(%r{[" |=/\\]}, '\\\\\0')
 
       when /^date(?::(?<format>.*))?$/
-        # TODO: validate how grafana handles multivariables with date format
+        # TODO: grafana does not seem to allow multiselection of variables with date format - raise an error if this happens anyway
         get_date_formatted(value, Regexp.last_match(1))
 
       when ''
         # default
-        if multi?
+        if multi? && value.is_a?(Array)
           value = value.map { |item| "'#{item.gsub(/'/, "''")}'" }
           return value.join(',')
         end
@@ -156,14 +162,13 @@ module Grafana
 
       else
         # glob and all unknown
-        # TODO add check for array value properly for all cases
         return "{#{value.join(',')}}" if multi? && value.is_a?(Array)
 
         value
       end
     end
 
-    # @return [Boolean] true, if the value can contain multiple selections, i.e. is an Array
+    # @return [Boolean] true, if the value can contain multiple selections, i.e. can contain an Array
     def multi?
       return @config['multi'] unless @config['multi'].nil?
 
@@ -197,6 +202,8 @@ module Grafana
       until work_string.empty?
         tmp = work_string.scan(/^(?:M{1,4}|D{1,4}|d{1,4}|e|E|w{1,2}|W{1,2}|Y{4}|Y{2}|A|a|H{1,2}|
                                     h{1,2}|k{1,2}|m{1,2}|s{1,2}|S+|X)/x)
+
+        # TODO: add test for sub! and switch to non-modifying frozen string action
         if tmp.empty?
           matches << work_string[0]
           work_string.sub!(/^#{work_string[0]}/, '')

data/lib/grafana_reporter/abstract_query.rb

@@ -1,23 +1,58 @@
 # frozen_string_literal: true
 
+require_relative 'abstract_table_format_strategy'
+require_relative 'csv_table_format_strategy'
+
 module GrafanaReporter
   # @abstract Override {#pre_process} and {#post_process} in subclass.
   #
   # Superclass containing everything for all queries towards grafana.
   class AbstractQuery
-    attr_accessor :datasource, :timeout, :from, :to
+    attr_accessor :datasource
     attr_writer :raw_query
-    attr_reader :variables, :result, :panel
+    attr_reader :variables, :result, :panel, :dashboard
+
+    def timeout
+      # TODO: PRIO check where value priorities should be evaluated
+      return @variables['timeout'].raw_value if @variables['timeout']
+      return @variables['grafana_default_timeout'].raw_value if @variables['grafana_default_timeout']
+
+      nil
+    end
+
+    # @param grafana_obj [Object] {Grafana::Grafana}, {Grafana::Dashboard} or {Grafana::Panel} object for which the query is executed
+    # @param opts [Hash] hash options, which may consist of:
+    # @option opts [Hash] :variables hash of variables, which shall be used to replace variable references in the query
+    # @option opts [Boolean] :ignore_dashboard_defaults True if {#assign_dashboard_defaults} should not be called
+    # @option opts [Boolean] :do_not_use_translated_times True if given from and to times should used as is, without being resolved to reporter times - using this parameter can lead to inconsistent report contents
+    def initialize(grafana_obj, opts = {})
+      if grafana_obj.is_a?(Grafana::Panel)
+        @panel = grafana_obj
+        @dashboard = @panel.dashboard
+        @grafana = @dashboard.grafana
+
+      elsif grafana_obj.is_a?(Grafana::Dashboard)
+        @dashboard = grafana_obj
+        @grafana = @dashboard.grafana
+
+      elsif grafana_obj.is_a?(Grafana::Grafana)
+        @grafana = grafana_obj
+
+      elsif !grafana_obj
+        # nil given
 
-    # @param grafana_or_panel [Object] {Grafana::Grafana} or {Grafana::Panel} object for which the query is executed
-    def initialize(grafana_or_panel)
-      if grafana_or_panel.is_a?(Grafana::Panel)
-        @panel = grafana_or_panel
-        @grafana = @panel.dashboard.grafana
       else
-        @grafana = grafana_or_panel
+        raise GrafanaReporterError, "Internal error in AbstractQuery: given object is of type #{grafana_obj.class.name}, which is not supported"
       end
       @variables = {}
+      @variables['from'] = Grafana::Variable.new(nil)
+      @variables['to'] = Grafana::Variable.new(nil)
+
+      assign_dashboard_defaults unless opts[:ignore_dashboard_defaults]
+      opts[:variables].each { |k, v| assign_variable(k, v) } if opts[:variables].is_a?(Hash)
+
+      @translate_times = true
+      @translate_times = false if opts[:do_not_use_translated_times]
     end
 
     # @abstract
@@ -31,11 +66,33 @@ module GrafanaReporter
     def execute
       return @result unless @result.nil?
 
+      from = @variables['from'].raw_value
+      to = @variables['to'].raw_value
+      if @translate_times
+        from = translate_date(@variables['from'], @variables['grafana_report_timestamp'], false, @variables['from_timezone'] ||
+                              @variables['grafana_default_from_timezone'])
+        to = translate_date(@variables['to'], @variables['grafana_report_timestamp'], true, @variables['to_timezone'] ||
+                            @variables['grafana_default_to_timezone'])
+      end
+
       pre_process
       raise DatasourceNotSupportedError.new(@datasource, self) if @datasource.is_a?(Grafana::UnsupportedDatasource)
 
-      @result = @datasource.request(from: @from, to: @to, raw_query: raw_query, variables: grafana_variables,
-                                    prepared_request: @grafana.prepare_request, timeout: timeout)
+      begin
+        @result = @datasource.request(from: from, to: to, raw_query: raw_query, variables: grafana_variables,
+                                      prepared_request: @grafana.prepare_request, timeout: timeout)
+      rescue ::Grafana::GrafanaError
+        # grafana errors will be directly passed through
+        raise
+      rescue GrafanaReporterError
+        # grafana errors will be directly passed through
+        raise
+      rescue StandardError => e
+        raise DatasourceRequestInternalError.new(@datasource, e.message)
+      end
+
+      raise DatasourceRequestInvalidReturnValueError.new(@datasource, @result) unless datasource_response_valid?
+
       post_process
       @result
     end
@@ -66,17 +123,6 @@ module GrafanaReporter
       raise NotImplementedError
     end
 
-    # Used to specify variables to be used for this query. This method ensures, that only the values of the
-    # {Grafana::Variable} stored in the +variables+ Array are overwritten.
-    # @param name [String] name of the variable to set
-    # @param variable [Grafana::Variable] variable from which the {Grafana::Variable#raw_value} will be assigned to the query variables
-    def assign_variable(name, variable)
-      raise GrafanaReporterError, "Provided variable is not of type Grafana::Variable (name: '#{name}', value: '#{value}')" unless variable.is_a?(Grafana::Variable)
-
-      @variables[name] ||= variable
-      @variables[name].raw_value = variable.raw_value
-    end
-
     # Transposes the given result.
     #
     # NOTE: Only the +:content+ of the given result hash is transposed. The +:header+ is ignored.
@@ -105,10 +151,10 @@ module GrafanaReporter
 
       filter_columns = filter_columns_variable.raw_value
       filter_columns.split(',').each do |filter_column|
-        pos = result[:header][0].index(filter_column)
+        pos = result[:header].index(filter_column)
 
         unless pos.nil?
-          result[:header][0].delete_at(pos)
+          result[:header].delete_at(pos)
           result[:content].each { |row| row.delete_at(pos) }
         end
       end
@@ -174,7 +220,6 @@ module GrafanaReporter
     # @param result [Hash] preformatted query result (see {Grafana::AbstractDatasource#request}.
     # @param configs [Array<Grafana::Variable>] one variable for replacing values in one column
     # @return [Hash] query result with replaced values
-    # TODO: make sure that caught errors are also visible in logger
     def replace_values(result, configs)
       return result if configs.empty?
 
@@ -189,8 +234,11 @@ module GrafanaReporter
 
           k = arr[0]
           v = arr[1]
-          k.gsub!(/\\([:,])/, '\1')
-          v.gsub!(/\\([:,])/, '\1')
+
+          # allow keys and values to contain escaped colons or commas
+          k = k.gsub(/\\([:,])/, '\1')
+          v = v.gsub(/\\([:,])/, '\1')
+
           result[:content].map do |row|
             (row.length - 1).downto 0 do |i|
               if cols.include?(i + 1) || cols.empty?
@@ -200,6 +248,7 @@ module GrafanaReporter
                   begin
                     row[i] = row[i].to_s.gsub(/#{k}/, v) if row[i].to_s =~ /#{k}/
                   rescue StandardError => e
+                    @grafana.logger.error(e.message)
                     row[i] = e.message
                   end
 
@@ -224,6 +273,7 @@ module GrafanaReporter
                                  end
                       end
                     rescue StandardError => e
+                      @grafana.logger.error(e.message)
                       row[i] = e.message
                     end
                   end
@@ -241,6 +291,34 @@ module GrafanaReporter
       result
     end
 
+    # Used to build a table output in a custom format.
+    # @param result [Hash] preformatted sql hash, (see {Grafana::AbstractDatasource#request})
+    # @param opts [Hash] options for the formatting:
+    # @option opts [Grafana::Variable] :row_divider requested row divider for the result table, only to be used with table_formatter `adoc_deprecated`
+    # @option opts [Grafana::Variable] :column_divider requested row divider for the result table, only to be used with table_formatter `adoc_deprecated`
+    # @option opts [Grafana::Variable] :include_headline specifies if table should contain headline, defaults to false
+    # @option opts [Grafana::Variable] :table_formatter specifies which formatter shall be used, defaults to 'csv'
+    # @return [String] table in custom output format
+    def format_table_output(result, opts)
+      opts = { include_headline: Grafana::Variable.new('false'),
+               table_formatter: Grafana::Variable.new('csv'),
+               row_divider: Grafana::Variable.new('| '),
+               column_divider: Grafana::Variable.new(' | ') }.merge(opts.delete_if {|_k, v| v.nil? })
+
+      if opts[:table_formatter].raw_value == 'adoc_deprecated'
+        @grafana.logger.warn("You are using deprecated 'table_formatter' named 'adoc_deprecated', which will be "\
+                             "removed in a future version. Start using 'adoc_plain' or register your own "\
+                             "implementation of AbstractTableFormatStrategy.")
+        return result[:content].map do |row|
+          opts[:row_divider].raw_value + row.map do |item|
+            item.to_s.gsub('|', '\\|')
+          end.join(opts[:column_divider].raw_value)
+        end.join("\n")
+      end
+
+      AbstractTableFormatStrategy.get(opts[:table_formatter].raw_value).format(result, opts[:include_headline].raw_value.downcase == 'true')
+    end
+
     # Used to translate the relative date strings used by grafana, e.g. +now-5d/w+ to the
     # correct timestamp. Reason is that grafana does this in the frontend, which we have
     # to emulate here for the reporter.
@@ -256,8 +334,10 @@ module GrafanaReporter
     # @param timezone [Grafana::Variable] timezone to use, if not system timezone
     # @return [String] translated date as timestamp string
     def translate_date(orig_date, report_time, is_to_time, timezone = nil)
-      # TODO: add test case for creation of variable, if not given, maybe also print a warning
+      @grafana.logger.warn("#translate_date has been called without 'report_time' - using current time as fallback.") unless report_time
       report_time ||= ::Grafana::Variable.new(Time.now.to_s)
+      orig_date = orig_date.raw_value if orig_date.is_a?(Grafana::Variable)
+
       return (DateTime.parse(report_time.raw_value).to_time.to_i * 1000).to_s unless orig_date
       return orig_date if orig_date =~ /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/
       return orig_date if orig_date =~ /^\d+$/
@@ -265,24 +345,24 @@ module GrafanaReporter
       # check if a relative date is mentioned
       date_spec = orig_date.clone
 
-      date_spec.slice!(/^now/)
+      date_spec = date_spec.gsub(/^now/, '')
       raise TimeRangeUnknownError, orig_date unless date_spec
 
       date = DateTime.parse(report_time.raw_value)
-      # TODO: allow from_translated or similar in ADOC template
+      # TODO: PRIO allow from_translated or similar in ADOC template
       date = date.new_offset(timezone.raw_value) if timezone
 
       until date_spec.empty?
         fit_match = date_spec.match(%r{^/(?<fit>[smhdwMy])})
         if fit_match
           date = fit_date(date, fit_match[:fit], is_to_time)
-          date_spec.slice!(%r{^/#{fit_match[:fit]}})
+          date_spec = date_spec.gsub(%r{^/#{fit_match[:fit]}}, '')
         end
 
         delta_match = date_spec.match(/^(?<op>(?:-|\+))(?<count>\d+)?(?<unit>[smhdwMy])/)
         if delta_match
           date = delta_date(date, "#{delta_match[:op]}#{delta_match[:count] || 1}".to_i, delta_match[:unit])
-          date_spec.slice!(/^#{delta_match[:op]}#{delta_match[:count]}#{delta_match[:unit]}/)
+          date_spec = date_spec.gsub(/^#{delta_match[:op]}#{delta_match[:count]}#{delta_match[:unit]}/, '')
         end
 
         raise TimeRangeUnknownError, orig_date unless fit_match || delta_match
@@ -296,6 +376,45 @@ module GrafanaReporter
 
     private
 
+    # Used to specify variables to be used for this query. This method ensures, that only the values of the
+    # {Grafana::Variable} stored in the +variables+ Array are overwritten.
+    # @param name [String] name of the variable to set
+    # @param variable [Grafana::Variable] variable from which the {Grafana::Variable#raw_value} will be assigned to the query variables
+    def assign_variable(name, variable)
+      variable = Grafana::Variable.new(variable) unless variable.is_a?(Grafana::Variable)
+
+      @variables[name] ||= variable
+      @variables[name].raw_value = variable.raw_value
+    end
+
+    # Sets default configurations from the given {Grafana::Dashboard} and store them as settings in the
+    # {AbstractQuery}.
+    #
+    # Following data is extracted:
+    # - +from+, by {Grafana::Dashboard#from_time}
+    # - +to+, by {Grafana::Dashboard#to_time}
+    # - and all variables as {Grafana::Variable}, prefixed with +var-+, as grafana also does it
+    def assign_dashboard_defaults
+      return unless @dashboard
+
+      assign_variable('from', @dashboard.from_time)
+      assign_variable('to', @dashboard.to_time)
+      @dashboard.variables.each { |item| assign_variable("var-#{item.name}", item) }
+    end
+
+    def datasource_response_valid?
+      return false if @result.nil?
+      return false unless @result.is_a?(Hash)
+      # TODO: compare how empty valid responses look like in grafana
+      return true if @result.empty?
+      return false unless @result.key?(:header)
+      return false unless @result.key?(:content)
+      return false unless @result[:header].is_a?(Array)
+      return false unless @result[:content].is_a?(Array)
+
+      true
+    end
+
     # @return [Hash<String, Variable>] all grafana variables stored in this query, i.e. the variable name
     #  is prefixed with +var-+
     def grafana_variables

data/lib/grafana_reporter/abstract_report.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module GrafanaReporter
-  # @abstract Override {#create_report} and {#progress}.
+  # @abstract Override {#build} and {#progress}.
   #
   # This class is used to build a report on basis of a given configuration and
   # template.
@@ -124,8 +124,9 @@ module GrafanaReporter
       logger.internal_messages
     end
 
-    # Is being called to start the report generation.
-    # @param template [String] path to the template to be used, trailing +.adoc+ extension may be omitted
+    # Is being called to start the report generation. To execute the specific report generation, this function
+    # calls the abstract {#build} method with the given parameters.
+    # @param template [String] path to the template to be used, trailing extension may be omitted, whereas {#default_template_extension} will be appended
     # @param destination_file_or_path [String or File] path to the destination report or file object to use
     # @param custom_attributes [Hash] custom attributes, which shall be merged with priority over the configuration
     # @return [void]
@@ -135,13 +136,31 @@ module GrafanaReporter
       @destination_file_or_path = destination_file_or_path
       @custom_attributes = custom_attributes
 
-      # automatically add extension, if a file with adoc extension exists
-      @template = "#{@template}.adoc" if File.file?("#{@template}.adoc") && !File.file?(@template.to_s)
+      # automatically add extension, if a file with default template extension exists
+      @template = "#{@template}.#{self.class.default_template_extension}" if File.file?("#{@template}.#{self.class.default_template_extension}") && !File.file?(@template.to_s)
       raise MissingTemplateError, @template.to_s unless File.file?(@template.to_s)
 
       notify(:on_before_create)
       @start_time = Time.new
       logger.info("Report started at #{@start_time}")
+      build
+    rescue MissingTemplateError => e
+      @logger.error(e.message)
+      @error = [e.message]
+      done!
+      raise e
+    rescue StandardError => e
+      # catch all errors during execution
+      died_with_error(e)
+      raise e
+    ensure
+      done!
+    end
+
+    # @abstract
+    # Needs to be overridden by the report implementation.
+    def build(template, destination_file_or_path, custom_attributes)
+      raise NotImplementedError
     end
 
     # Used to calculate the progress of a report. By default expects +@total_steps+ to contain the total
@@ -167,6 +186,18 @@ module GrafanaReporter
       raise NotImplementedError
     end
 
+    # @abstract
+    # @return [String] specifying the default extension of a template file
+    def self.default_template_extension
+      raise NotImplementedError
+    end
+
+    # @abstract
+    # @return [String] specifying the default extension of a rendered result file
+    def self.default_result_extension
+      raise NotImplementedError
+    end
+
     private
 
     # Called, if the report generation has died with an error.
@@ -188,6 +219,7 @@ module GrafanaReporter
     def done!
       return if @done
 
+      @destination_file_or_path.close if @destination_file_or_path.is_a?(File)
       @done = true
       @end_time = Time.new
       @start_time ||= @end_time