ruby-grafana-reporter 0.1.6

data/lib/grafana_reporter/asciidoctor/report.rb

@@ -0,0 +1,159 @@
+module GrafanaReporter
+  module Asciidoctor
+    # Implementation of a specific {AbstractReport}. It is used to
+    # build reports specifically for asciidoctor results.
+    class Report < GrafanaReporter::AbstractReport
+      # (see AbstractReport#initialize)
+      def initialize(config, template, destination_file_or_path = nil, custom_attributes = {})
+        super
+        @current_pos = 0
+        @image_files = []
+        @grafana_instances = {}
+      end
+
+      # Starts to create an asciidoctor report. It utilizes all {Extensions} to
+      # realize the conversion.
+      # @see AbstractReport#create_report
+      # @return [void]
+      def create_report
+        @start_time = Time.now
+        attrs = {'convert-backend' => 'pdf'}.merge(@config.default_document_attributes.merge(@custom_attributes))
+        attrs['grafana-report-timestamp'] = @start_time.to_s
+        logger.info('Report started at ' + @start_time.to_s)
+        logger.debug('Document attributes: ' + attrs.to_s)
+
+        initialize_step_counter
+
+        # register necessary extensions for the current report
+        ::Asciidoctor::LoggerManager.logger = logger
+
+        registry = ::Asciidoctor::Extensions::Registry.new
+        #TODO dynamically register macros, which is also needed when supporting custom macros
+        registry.inline_macro Extensions::PanelImageInlineMacro.new.current_report(self)
+        registry.inline_macro Extensions::PanelQueryValueInlineMacro.new.current_report(self)
+        registry.inline_macro Extensions::PanelPropertyInlineMacro.new.current_report(self)
+        registry.inline_macro Extensions::SqlValueInlineMacro.new.current_report(self)
+        registry.block_macro Extensions::PanelImageBlockMacro.new.current_report(self)
+        registry.include_processor Extensions::ValueAsVariableIncludeProcessor.new.current_report(self)
+        registry.include_processor Extensions::PanelQueryTableIncludeProcessor.new.current_report(self)
+        registry.include_processor Extensions::SqlTableIncludeProcessor.new.current_report(self)
+        registry.include_processor Extensions::ShowEnvironmentIncludeProcessor.new.current_report(self)
+        registry.include_processor Extensions::ShowHelpIncludeProcessor.new.current_report(self)
+        registry.include_processor Extensions::AnnotationsTableIncludeProcessor.new.current_report(self)
+        registry.include_processor Extensions::AlertsTableIncludeProcessor.new.current_report(self)
+
+        ::Asciidoctor.convert_file(@template, extension_registry: registry, backend: attrs['convert-backend'], to_file: path, attributes: attrs, header_footer: true)
+
+        @destination_file_or_path.close if @destination_file_or_path.is_a?(File)
+
+        # store report including als images as ZIP file, if the result is not a PDF
+        # TODO add tests for zipping results
+        if attrs['convert-backend'] != 'pdf'
+          dest_path = @destination_file_or_path
+          dest_path = @destination_file_or_path.path if @destination_file_or_path.is_a?(File)
+
+          # build zip file
+          zip_file = Tempfile.new("gf_zip")
+          Zip::File.open(zip_file.path, Zip::File::CREATE) do |zipfile|
+            # add report file
+            zipfile.get_output_stream(dest_path.gsub(@config.reports_folder, '') + ".#{attrs['convert-backend']}") { |f| f.puts File.read(dest_path) }
+
+            # add image files
+            @image_files.each do |file|
+              zipfile.get_output_stream(file.path.gsub(@config.images_folder, '')) { |f| f.puts File.read(file.path) }
+            end
+          end
+
+          # replace original file with zip file
+          zip_file.rewind
+          begin
+            File.write(dest_path, zip_file.read)
+          rescue => e
+            logger.fatal("Could not overwrite report file '#{dest_path}' with ZIP file. (#{e.message}).")
+          end
+
+          # cleanup temporary zip file
+          zip_file.close
+          zip_file.unlink
+        end
+        
+        clean_image_files
+        @end_time = Time.now
+        logger.info('Report finished after ' + (@end_time - @start_time).to_s + ' seconds.')
+        @done = true
+      rescue StandardError => e
+        # catch all errors during execution
+        died_with_error(e)
+        raise e
+      end
+
+      # @see AbstractReport#progress
+      # @return [Float] number between 0 and 1 reflecting the current progress.
+      def progress
+        return 0 if @total_steps.to_i.zero?
+
+        @current_pos.to_f / @total_steps
+      end
+
+      # @param instance [String] requested grafana instance
+      # @return [Grafana::Grafana] the requested grafana instance.
+      def grafana(instance)
+        unless @grafana_instances[instance]
+          @grafana_instances[instance] = ::Grafana::Grafana.new(@config.grafana_host(instance), @config.grafana_api_key(instance), logger: @logger, datasources: @config.grafana_datasources(instance))
+        end
+        @grafana_instances[instance]
+      end
+
+      # Increments the progress.
+      # @return [Integer] number of the current progress position.
+      def next_step
+        @current_pos += 1
+        @current_pos
+      end
+
+      # Called to save a temporary image file. After the final generation of the
+      # report, these temporary files will automatically be removed.
+      # @param img_data [String] image file raw data, which shall be saved
+      # @return [String] path to the temporary file.
+      def save_image_file(img_data)
+        file = Tempfile.new(['gf_image_', '.png'], @config.images_folder.to_s)
+        file.write(img_data)
+        path = file.path.gsub(/#{@config.images_folder}/, '')
+
+        @image_files << file
+        file.close
+
+        path
+      end
+
+      # Called, if the report generation has died with an error.
+      # @param e [StandardError] occured error
+      # @return [void]
+      def died_with_error(e)
+        @error = [e.message] << [e.backtrace]
+        @end_time = Time.now
+        @done = true
+      end
+
+      private
+
+      def clean_image_files
+        @image_files.each(&:unlink)
+        @image_files = []
+      end
+      
+      def initialize_step_counter
+        @total_steps = 0
+        File.readlines(@template).each do |line|
+          begin
+            @total_steps += line.gsub(%r{//.*}, '').scan(/(?:grafana_panel_image|grafana_panel_query_value|grafana_panel_query_table|grafana_sql_value|grafana_sql_table|grafana_environment|grafana_help|grafana_panel_property|grafana_annotations|grafana_alerts|grafana_value_as_variable)/).length
+          rescue => e
+            logger.error("Could not process line '#{line}' (Error: #{e.message})")
+            raise e
+          end
+        end
+        logger.debug("Template #{@template} contains #{@total_steps.to_s} calls of grafana reporter functions.")
+      end
+    end
+  end
+end

data/lib/grafana_reporter/asciidoctor/sql_first_value_query.rb

@@ -0,0 +1,34 @@
+module GrafanaReporter
+  module Asciidoctor
+    # This class is being used to execute a SQL query against a grafana datasource.
+    # Only the first result in the first column will be returned as a single value.
+    class SqlFirstValueQuery < Grafana::AbstractSqlQuery
+      include QueryMixin
+
+      # Executes {QueryMixin#format_columns}, {QueryMixin#replace_values} and
+      # {QueryMixin#filter_columns} on the query results.
+      #
+      # Finally only the first value in the first row and the first column of
+      # will be returned.
+      # @return [void]
+      def post_process
+        results = preformat_sql_result(@result.body)
+        results = format_columns(results, @variables['format'])
+        results = replace_values(results, @variables.select { |k, _v| k =~ /^replace_values_\d+/ })
+        results = filter_columns(results, @variables['filter_columns'])
+        if @variables['filter_column']
+          @report.logger.warn("DEPRECATED: Call of  no longer supported function 'filter_column' has been found. Rename to 'filter_columns'")
+          results = filter_columns(results, @variables['filter_column'])
+        end
+
+        unless results[:content].empty?
+          unless results[:content][0].empty?
+            @result = results[:content][0][0]
+            return
+          end
+        end
+        @result = ''
+      end
+    end
+  end
+end

data/lib/grafana_reporter/asciidoctor/sql_table_query.rb

@@ -0,0 +1,32 @@
+module GrafanaReporter
+  module Asciidoctor
+    # This class is being used to execute a SQL query against a grafana datasource.
+    # The results will be formatted as as asciidoctor table.
+    class SqlTableQuery < Grafana::AbstractSqlQuery
+      include QueryMixin
+
+      # Executes {QueryMixin#format_columns}, {QueryMixin#replace_values} and
+      # {QueryMixin#filter_columns} on the query results.
+      #
+      # Finally the results are formatted as a asciidoctor table.
+      # @return [void]
+      def post_process
+        results = preformat_sql_result(@result.body)
+        results = format_columns(results, @variables['format'])
+        results = replace_values(results, @variables.select { |k, _v| k =~ /^replace_values_\d+/ })
+        results = filter_columns(results, @variables['filter_columns'])
+        if @variables['filter_column']
+          @report.logger.warn("DEPRECATED: Call of  no longer supported function 'filter_column' has been found. Rename to 'filter_columns'")
+          results = filter_columns(results, @variables['filter_column'])
+        end
+        results = transpose(results, @variables['transpose'])
+        row_divider = '| '
+        row_divider = @variables['row_divider'].raw_value if @variables['row_divider'].is_a?(Grafana::Variable)
+        column_divider = ' | '
+        column_divider = @variables['column_divider'].raw_value if @variables['column_divider'].is_a?(Grafana::Variable)
+
+        @result = results[:content].map { |row| row_divider + row.map { |item| column_divider == ' | ' ? item.to_s.gsub('|', '\\|') : item.to_s }.join(column_divider) }
+      end
+    end
+  end
+end

data/lib/grafana_reporter/configuration.rb

@@ -0,0 +1,326 @@
+# In this namespace all objects needed for the grafana reporter are collected.
+module GrafanaReporter
+  # Used to store the whole settings, which are necessary to run the reporter.
+  # It can read configuration files, but might also be configured programmatically.
+  #
+  # This class also contains a function {#validate}, which ensures that the
+  # provided settings are set properly.
+  #
+  # Using this class is embedded in the {Application::Application#configure_and_run}.
+  #
+  # TODO add config example
+  class Configuration
+    # @return [AbstractReport] specific report class, which should be used.
+    attr_accessor :report_class
+
+    # Returned by {#mode} if only a connection test shall be executed.
+    MODE_CONNECTION_TEST = 'test'
+    # Returned by {#mode} if only one configured report shall be rendered.
+    MODE_SINGLE_RENDER = 'single-render'
+    # Returned by {#mode} if the default webservice shall be started.
+    MODE_SERVICE = 'webservice'
+
+    def initialize
+      @config = {}
+      @logger = ::Logger.new(STDERR, level: :unknown)
+      # TODO: set report class somewhere else, but make it known here
+      self.report_class = Asciidoctor::Report
+    end
+
+    attr_reader :logger
+
+    # @return [String] mode, in which the reporting shall be executed. One of {MODE_CONNECTION_TEST}, {MODE_SINGLE_RENDER} and {MODE_SERVICE}.
+    def mode
+      return MODE_SERVICE if get_config('grafana-reporter:run-mode') != MODE_CONNECTION_TEST and get_config('grafana-reporter:run-mode') != MODE_SINGLE_RENDER
+
+      return get_config('grafana-reporter:run-mode')
+    end
+
+    # @return [String] configured report template. Only needed in {MODE_SINGLE_RENDER}.
+    def template
+      get_config('default-document-attributes:var-template')
+    end
+
+    # @return [String] destination filename for the report in {MODE_SINGLE_RENDER}.
+    def to_file
+      return get_config('to_file') || true if mode == MODE_SINGLE_RENDER
+
+      get_config('to_file')
+    end
+
+    # @return [Array<String>] names of the configured grafana_instances.
+    def grafana_instances
+      instances = get_config('grafana')
+      instances.keys
+    end
+
+    # @param instance [String] grafana instance name, for which the value shall be retrieved.
+    # @return [String] configured 'host' for the requested grafana instance.
+    def grafana_host(instance = 'default')
+      host = get_config("grafana:#{instance}:host")
+      raise GrafanaInstanceWithoutHostError, instance if host.nil?
+
+      host
+    end
+
+    # @param instance [String] grafana instance name, for which the value shall be retrieved.
+    # @return [String] configured 'api_key' for the requested grafana instance.
+    def grafana_api_key(instance = 'default')
+      get_config("grafana:#{instance}:api_key")
+    end
+
+    # @param instance [String] grafana instance name, for which the value shall be retrieved.
+    # @return [Hash<String,Integer>] configured datasources for the requested grafana instance. Name as key, ID as value.
+    def grafana_datasources(instance = 'default')
+      hash = get_config("grafana:#{instance}:datasources")
+      return nil if hash.nil?
+
+      hash.map { |k, v| [k, v] }.to_h
+    end
+
+    # @return [String] configured folder, in which the report templates are stored including trailing slash. By default: current folder.
+    def templates_folder
+      result = get_config('grafana-reporter:templates-folder') || '.'
+      result.sub!(%r{[/]*$}, '/') unless result.empty?
+      result
+    end
+
+    # Returns configured folder, in which temporary images during report generation
+    # shall be stored including trailing slash. Folder has to be a subfolder of
+    # {#templates_folder}. By default: current folder.
+    # @return [String] configured folder, in which temporary images shall be stored.
+    def images_folder
+      img_path = templates_folder
+      img_path = img_path.empty? ? get_config('default-document-attributes:imagesdir').to_s : img_path + get_config('default-document-attributes:imagesdir').to_s
+      img_path.empty? ? './' : img_path.sub(%r{[/]*$}, '/')
+    end
+
+    # @return [String] name of grafana instance, against which a test shall be executed
+    def test_instance
+      get_config('grafana-reporter:test-instance')
+    end
+
+    # @return [String] configured folder, in which the reports shall be stored including trailing slash. By default: current folder.
+    def reports_folder
+      result = get_config('grafana-reporter:reports-folder') || '.'
+      result.sub!(%r{[/]*$}, '/') unless result.empty?
+      result
+    end
+
+    # @return [Integer] how many hours a generated report shall be retained, before it shall be deleted. By default: 24.
+    def report_retention
+      get_config('grafana-reporter:report-retention') || 24
+    end
+
+    # @return [Integer] port, on which the webserver shall run. By default: 8815.
+    def webserver_port
+      get_config('grafana-reporter:webservice-port') || 8815
+    end
+
+    # The configuration made with the setting 'default-document-attributes' will
+    # be passed 1:1 to the asciidoctor report service. It can be used to preconfigure
+    # whatever is essential for the needed report renderings.
+    # @return [Hash] configured document attributes
+    def default_document_attributes
+      get_config('default-document-attributes') || {}
+    end
+
+    # Used to load the configuration of a file or a manually created Hash to this
+    # object. To make sure, that the configuration is valid, call {#validate}.
+    #
+    # NOTE: This function overwrites all existing configurations
+    # @param hash [Hash] configuration settings
+    # @return [void]
+    def load_config(hash)
+      @config = hash
+    end
+
+    # Used to do the configuration by a command line call. Therefore also help will
+    # be shown, in case no parameter has been given.
+    # @param params [Array<String>] command line parameters, mainly ARGV can be used.
+    # @return [Integer] 0 if everything is fine, -1 if execution shall be aborted.
+    def configure_by_command_line(params = [])
+      params << '--help' if params.empty?
+
+      parser = OptionParser.new do |opts|
+        opts.banner = 'Usage: ruby ruby-grafana-reporter.rb CONFIG_FILE [options]'
+
+        opts.on('-d', '--debug LEVEL', 'Specify detail level: FATAL, ERROR, WARN, INFO, DEBUG.') do |level|
+          @logger.level = Object.const_get("::Logger::Severity::#{level}") if level =~ /(?:FATAL|ERROR|WARN|INFO|DEBUG)/
+        end
+
+        opts.on('--test GRAFANA_INSTANCE', 'test current configuration against given GRAFANA_INSTANCE') do |instance|
+          if get_config('grafana-reporter')
+            @config['grafana-reporter']['run-mode'] = 'test'
+          else
+            @config.merge!({'grafana-reporter' => {'run-mode' => 'test'} })
+	  end
+          @config['grafana-reporter']['test-instance'] = instance
+        end
+
+        opts.on('-t', '--template TEMPLATE', 'Render a single ASCIIDOC template to PDF and exit') do |template|
+          if get_config('grafana-reporter')
+            @config['grafana-reporter']['run-mode'] = 'single-render'
+          else
+            @config.merge!({'grafana-reporter' => {'run-mode' => 'single-render'} })
+	  end
+          @config['default-document-attributes']['var-template'] = template
+        end
+
+        opts.on('-o', '--output FILE', 'Output filename if only a single file is rendered') do |file|
+          @config.merge!({ 'to_file' => file })
+        end
+
+        opts.on('-v', '--version', 'Version information') do
+          puts GRAFANA_REPORTER_VERSION.join('.')
+          return -1
+        end
+
+        opts.on('-h', '--help', 'Show this message') do
+          puts opts
+          return -1
+        end
+      end
+
+      unless params.empty?
+        if File.exist?(params[0])
+          config_file = params.slice!(0)
+          begin
+            load_config(YAML.load_file(config_file))
+          rescue StandardError => e
+            raise ConfigurationError, "Could not read CONFIG_FILE '#{config_file}' (Error: #{e.message})"
+          end
+        end
+      end
+      parser.parse!(params)
+
+      0
+    end
+
+    # This function shall be called, before the configuration object is used in the
+    # {Application::Application#run}. It ensures, that everything is setup properly
+    # and all necessary folders exist. Appropriate errors are raised in case of errors.
+    # @return [void]
+    def validate
+      validate_schema(schema, @config)
+
+      # check if set folders exist
+      raise FolderDoesNotExistError.new(reports_folder, 'reports-folder') unless File.directory?(reports_folder)
+      raise FolderDoesNotExistError.new(templates - folder, 'templates-folder') unless File.directory?(templates_folder)
+      raise FolderDoesNotExistError.new(images - folder, 'images-folder') unless File.directory?(images_folder)
+    end
+
+    private
+
+    def get_config(path)
+      return if path.nil?
+
+      cur_pos = @config
+      path.split(':').each do |subpath|
+        cur_pos = cur_pos[subpath] if cur_pos
+      end
+      cur_pos
+    end
+
+    def validate_schema(schema, subject)
+      return nil if subject.nil?
+
+      schema.each do |key, config|
+        type, min_occurence, next_level = config
+
+        validate_schema(next_level, subject[key]) if next_level
+
+        if key.nil?
+          # apply to all on this level
+          if subject.is_a?(Hash)
+            if subject.length < min_occurence
+              raise ConfigurationDoesNotMatchSchemaError.new(key, 'occur', min_occurence, subject.length)
+            end
+
+            subject.each do |k, _v|
+              sub_scheme = {}
+              sub_scheme[k] = schema[nil]
+              validate_schema(sub_scheme, subject)
+            end
+
+          elsif subject.is_a?(Array)
+            if subject.length < min_occurence
+              raise ConfigurationDoesNotMatchSchemaError.new(key, 'occur', min_occurence, subject.length)
+            end
+
+            subject.each_index do |i|
+              sub_scheme = {}
+              sub_scheme[i] = schema[nil]
+              validate_schema(sub_scheme, subject)
+            end
+
+          else
+            raise ConfigurationError, "Unhandled configuration data type '#{subject.class}'."
+          end
+        else
+          # apply to single item
+          if subject.is_a?(Hash)
+            if !subject.key?(key) && (min_occurence > 0)
+              raise ConfigurationDoesNotMatchSchemaError.new(key, 'occur', min_occurence, 0)
+            end
+            if !subject[key].is_a?(type) && subject.key?(key)
+              raise ConfigurationDoesNotMatchSchemaError.new(key, 'be a', type, subject[key].class)
+            end
+
+          elsif subject.is_a?(Array)
+            if (subject.length < key) && (min_occurence > subject.length)
+              raise ConfigurationDoesNotMatchSchemaError.new(key, 'occur', min_occurence, subject.length)
+            end
+            if !subject[key].is_a?(type) && (subject.length >= key)
+              raise ConfigurationDoesNotMatchSchemaError.new(key, 'be a', type, subject[key].class)
+            end
+
+          else
+            raise ConfigurationError, "Unhandled configuration data type '#{subject.class}'."
+          end
+        end
+      end
+
+      # validate also if subject has further configurations, which are not known by the reporter
+      subject.each do |item, subitems|
+        schema_config = schema[item] || schema[nil]
+        if schema_config.nil?
+          logger.warn("Item '#{item}' in configuration is unknown to the reporter and will be ignored")
+        end
+      end
+    end
+
+    def schema
+      {
+        'grafana' =>
+         [
+           Hash, 1,
+           {
+             nil =>
+              [
+                Hash, 1,
+                {
+                  'host' => [String, 1],
+                  'api_key' => [String, 0],
+                  'datasources' => [Hash, 0, { nil => [Integer, 1] }]
+                }
+              ]
+           }
+         ],
+        'default-document-attributes' => [Hash, 0],
+        'grafana-reporter' =>
+        [
+          Hash, 0,
+          {
+            'run-mode' => [String, 0],
+            'test-instance' => [String, 0],
+            'templates-folder' => [String, 0],
+            'reports-folder' => [String, 0],
+            'report-retention' => [Integer, 0],
+            'webservice-port' => [Integer, 0]
+          }
+        ]
+      }
+    end
+  end
+end