fluent-auditify 0.1.0

data/lib/fluent/auditify/plugin/conf_v1dupid.rb

@@ -0,0 +1,46 @@
+require 'fluent/config/error'
+require 'fluent/config/v1_parser'
+require 'fluent/auditify/plugin/conf'
+require 'fluent/auditify/parser/v1config'
+
+module Fluent::Auditify::Plugin
+  class V1DuplicatedId < Conf
+    Fluent::Auditify::Plugin.register_conf('v1dupid', self)
+
+    #
+    # The duplicated @id is detected by default, but treated as
+    # Fluent::ConfigError and does not show problematic location
+    # in configuration file.
+    # This plugin demonstrate how to implement simple plugin
+    #
+
+    def supported_platform?
+      :any
+    end
+
+    def parse(conf, options={})
+      begin
+        content = file_get_contents(conf)
+        root = Fluent::Config::V1Parser.parse(content, conf)
+        ids = []
+        root.elements.collect do |element|
+          ids << element["@id"]
+        end
+        duplicated_ids = ids.uniq.select do |id|
+          ids.count(id) > 1
+        end
+        duplicated_ids.each do |id|
+          file_readlines_each(conf) do |line, index|
+            next unless line.split.size == 2
+            if line.split == ["@id", id]
+              guilty(:error, "#{id} is duplicated",
+                     {path: conf, line: index + 1, content: line.chomp, category: :syntax, plugin: :v1dupid})
+            end
+          end
+        end
+      rescue => e
+        log.error("parse error: #{e.message}")
+      end
+    end
+  end
+end

data/lib/fluent/auditify/plugin.rb

@@ -0,0 +1,65 @@
+require 'fluent/config/error'
+require 'fluent/auditify/registry'
+require 'logger'
+
+module Fluent
+  module Auditify
+    module Plugin
+
+      # e.g. PATH_TO_...lib/fluent/auditify/plugin
+      DEFAULT_PLUGIN_PATH = File.expand_path('../plugin', __FILE__)
+      # e.g. PATH_TO_...lib/fluent/auditify
+      FLUENT_AUDITIFY_LIB_PATH = File.dirname(DEFAULT_PLUGIN_PATH)
+
+      CONF_REGISTRY = Registry.new(:conf, 'fluent/auditify/conf_')
+
+      REGISTRIES = [CONF_REGISTRY]
+
+      CHARGES = []
+
+      ARTIFACT = []
+
+      def self.charges
+        CHARGES
+      end
+
+      def self.registries
+        REGISTRIES
+      end
+
+      def self.discard
+        CHARGES.pop(CHARGES.size)
+      end
+
+      # This method will be executed when require it
+      def self.register_conf(plugin_name, plugin_klass)
+        caller_with_location = caller_locations(1, 1).first
+        if caller_with_location.path.include?('/lib/fluent/auditify/plugin')
+          gem_name = File.basename(File.dirname(caller_with_location.path.sub('/lib/fluent/auditify/plugin', '')))
+          plugin_name = "#{plugin_name}@#{gem_name}"
+        else
+          raise Fluent::ConfigError, "Fluent Auditify plugin must be installed as gem: <#{plugin_name}>."
+        end
+
+        if !plugin_klass.is_a?(Class) and
+          !([:supported_platform?, :parse].all? { |v| plugin_klass.respond_to?(v) })
+          raise Fluent::ConfigError, "Invalid Fluent Auditify plugin implementation as 'conf' plugin: <#{plugin_name}>."
+        end
+
+        CONF_REGISTRY.register(:conf, plugin_name, plugin_klass)
+      end
+
+      def self.guilty(level, message, options={})
+        CHARGES.push([level, message, options])
+      end
+
+      def self.polish(object)
+        ARTIFACT.push(object)
+      end
+
+      def self.artifact
+        ARTIFACT.pop
+      end
+    end
+  end
+end

data/lib/fluent/auditify/plugin_manager.rb

@@ -0,0 +1,178 @@
+require 'fluent/auditify/plugin'
+require 'fluent/auditify/reporter'
+require 'fluent/auditify/reporter'
+require 'tmpdir'
+
+module Fluent
+  module Auditify
+    class PluginManager
+      include Plugin
+
+      def initialize(logger = nil, mask_only: false)
+        @logger = logger
+        @plugins = []
+        @mask_only = mask_only
+        load
+        Fluent::Auditify::Plugin.registries.each do |registry|
+          registry.map.each do |sym, klass|
+            @logger.debug("Instantiate #{klass} for <#{sym}> plugin") if @logger
+            @plugins << klass.new
+          end
+        end
+      end
+
+      # search plugin with prefix
+      def self.search(plugin_name, logger = nil)
+        # Find plugin and require it in advance
+
+        # Lastly, load built-in plugins
+        plugin_path = File.expand_path(File.join(FLUENT_AUDITIFY_LIB_PATH,
+                                                 'fluent/auditify/plugin/conf_',
+                                                 "#{plugin_name}.rb"))
+        if File.exist?(plugin_path)
+          logger.debug("Loading <#{plugin_path}>") if logger
+          require plugin_path
+          return
+        end
+      end
+
+      def load
+        builtin_plugin_paths.each do |plugin_path|
+          @logger.debug("Loading <#{plugin_path}>") if @logger
+          if @mask_only
+            if plugin_path.include?('/lib/fluent/auditify/plugin/conf_mask_secrets.rb')
+              require plugin_path
+            end
+          else
+            require plugin_path
+          end
+        end
+      end
+
+      def builtin_plugin_paths
+        Dir.glob("#{DEFAULT_PLUGIN_PATH}/*.rb").select do |path|
+          File.dirname(path) == DEFAULT_PLUGIN_PATH and
+            File.basename(path).start_with?('conf_')
+        end
+      end
+
+      def windows?
+        %w(mswin mingw).any? { |v| RbConfig::CONFIG['host_os'].include?(v) }
+      end
+
+      def linux?
+        RbConfig::CONFIG['host_os'].include?('linux')
+      end
+
+      def supported_plugin?(plugin)
+        unless plugin.respond_to?(:supported_platform?)
+          @logger.info("Plugin: <#{plugin.class}> must implement supported_platform?")
+          return false
+        end
+        platform = plugin.supported_platform?
+        case platform
+        when :windows
+          unless windows?
+            @logger.debug("Plugin: <#{plugin.class}> does not support #{RbConfig::CONFIG['host_os']}")
+            return false
+          end
+        when :linux
+          unless linux?
+            @logger.debug("Plugin: <#{plugin.class}> does not support #{RbConfig::CONFIG['host_os']}")
+            return false
+          end
+        else
+          # :any
+          true
+        end
+      end
+
+      def skip_plugin?(plugin)
+        unless supported_plugin?(plugin)
+          return true
+        end
+        unless plugin.respond_to?(:parse)
+          return true
+        end
+        unless plugin.respond_to?(:supported_file_extension?)
+          return true
+        end
+        unless plugin.respond_to?(:disabled?)
+          return true
+        end
+        false
+      end
+
+      def collect_related_config_files(object)
+        files = []
+        object.each do |directive|
+          if directive[:include]
+            files << directive[:include_path].to_s
+          elsif directive[:empty_line]
+            next
+          else
+            directive[:body].each do |element|
+              if element[:value] and element[:name].to_s == '@include'
+                files << element[:value].to_s
+              end
+            end
+          end
+        end
+        files
+      end
+
+      def evacuate(options={})
+        @workspace_dir = Dir.mktmpdir('fluent-auditify')
+        @base_dir = File.dirname(options[:config])
+        parser = Fluent::Auditify::Parser::V1ConfigParser.new
+        object = parser.parse(File.read(options[:config]))
+
+        # copy configuration files into workspace
+        touched = [options[:config]]
+        touched << collect_related_config_files(object).collect { |v| File.join(@base_dir, v) }
+        touched.flatten!
+        FileUtils.cp(touched, @workspace_dir)
+      end
+
+      def dispatch(options={})
+        evacuate(options)
+        @plugins.each do |plugin|
+          next if skip_plugin?(plugin)
+
+          config_path = File.join(@workspace_dir, File.basename(options[:config]))
+          ext = plugin.supported_file_extension?
+          ext_symbol = File.extname(config_path).delete('.').to_sym
+          unless ext.any?(ext_symbol)
+            @logger.debug("#{plugin.class} does not support #{config_path}")
+            next
+          end
+
+          plugin.instance_variable_set(:@log, @logger)
+          begin
+            @logger.debug { "#{plugin.class}\#parse" }
+            plugin.parse(config_path, options)
+
+            if plugin.respond_to?(:transform)
+              tree = plugin.transform(config_path, options)
+              util = Fluent::Auditify::ParsletUtil.new
+              util.export(tree, options)
+              @logger.info("Configuration files were saved at: #{@workspace_dir}")
+            end
+          rescue => e
+            @logger.error("#{e.message}")
+          end
+        end
+      end
+
+      def report(type)
+        case type
+        when :console
+          reporter = Fluent::Auditify::Reporter::ConsoleReporter.new(@logger)
+        when :json
+          reporter Fluent::Auditify::Reporter::JsonReporter.new(@logger)
+        end
+        reporter.run(Plugin.charges)
+      end
+    end
+  end
+end

data/lib/fluent/auditify/registry.rb

@@ -0,0 +1,35 @@
+require 'fluent/auditify'
+
+module Fluent
+  module Auditify
+    class Registry
+
+      attr_reader :map
+
+      def initialize(kind, prefix)
+        @kind = kind
+        # e.g. lib/fluent/auditify/conf_
+        @search_prefix = prefix
+        @map = {}
+      end
+
+      # @param [String] name plugin name
+      # @return [Class] the object which inherit from Fluent::Auditify::Plugin
+      def lookup(name)
+        sym = "#{@kind}/#{name}".to_sym
+        return @map[sym] if @map[sym]
+        raise NotFoundPluginError.new("Unknown #{@kind} plugin", name: name)
+      end
+
+      # @param [String] plugin_name plugin name without plugin type prefix
+      # @param [Class] plugin_klass inherit from Fluent::Auditify::Plugin
+      def register(kind, plugin_name, plugin_klass)
+        sym = "#{kind}/#{plugin_name}".to_sym
+        if @map.key?(sym)
+          raise DuplicatedPluginError.new("#{sym} is already registered")
+        end
+        @map[sym] = plugin_klass
+      end
+    end
+  end
+end

data/lib/fluent/auditify/reporter/console.rb

@@ -0,0 +1,70 @@
+require 'diff/lcs'
+require 'pastel'
+
+module Fluent
+  module Auditify
+    module Reporter
+      class ConsoleReporter
+        def initialize(logger, options={})
+          @options = {
+            format: :auto
+          }
+          @options.merge!(options)
+          @logger = logger
+        end
+
+        def bomb
+          "\u{1f4a3}"
+        end
+
+        def file_get_contents(path)
+          File.open(path) do |f| f.readlines end
+        end
+
+        def run(charges, logger=nil)
+          charges.each do |entry|
+            message = entry.first
+            options = entry.last
+            if options[:line] and options[:content]
+              if @options[:format] == :auto
+                lines = file_get_contents(options[:path])
+                min = options[:line] - 2 > 0 ? options[:line] - 2 : 0
+                max = options[:line] + 2 < lines.size ? options[:line] + 2 : lines.size - 1
+                content = ""
+                suggested = ""
+                min.upto(max).each_with_index do |line, index|
+                  content << "#{min + index + 1}: #{lines[min + index].chomp}\n"
+                  if options[:suggest] and options[:line] == min + index + 1
+                    suggested << "#{min + index + 1}: #{options[:suggest].chomp}\n"
+                  else
+                    suggested << "#{min + index + 1}: #{lines[min + index].chomp}\n"
+                  end
+                end
+                if options[:suggest]
+                  diff_content = ''
+                  Diff::LCS.sdiff(content.chars, suggested.chars).each do |change|
+                    case change.action
+                    when '-'
+                      diff_content << Pastel.new.red(change.old_element)
+                    when '+'
+                      diff_content << Pastel.new.green(change.new_element)
+                    else
+                      diff_content << change.old_element
+                    end
+                  end
+                  @logger.error("#{bomb} [plugin:#{options[:plugin]},category:#{options[:category]}] #{message} at #{options[:path]}:#{options[:line]}\n#{diff_content}")
+                else
+                  @logger.error("#{bomb} [plugin:#{options[:plugin]},category:#{options[:category]}] #{message} at #{options[:path]}:#{options[:line]}\n#{content}")
+                end
+              else
+                @logger.error("#{bomb} [plugin:#{options[:plugin]},category:#{options[:category]}] #{message} at #{options[:path]}:#{options[:line]}: #{options[:content]}")
+              end
+            else
+              @logger.error("#{bomb} [plugin:#{options[:plugin]},category:#{options[:category]}] #{message} at #{options[:path]}")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/auditify/reporter/json.rb

@@ -0,0 +1,14 @@
+module Fluent
+  module Auditify
+    module Reporter
+      class JsonReporter
+        def initialize(logger)
+          @logger = logger
+        end
+
+        def run(charges)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/auditify/reporter.rb

@@ -0,0 +1,2 @@
+require 'fluent/auditify/reporter/console'
+require 'fluent/auditify/reporter/json'

data/lib/fluent/auditify/syntax_checker.rb

@@ -0,0 +1,30 @@
+require 'fluent/auditify/plugin/base'
+require 'fluent/auditify/plugin_manager'
+require 'fluent/auditify/log'
+
+module Fluent
+  module Auditify
+    class SyntaxChecker
+      def initialize(options={})
+        log_options = {
+          log_level: options[:log_level] || Logger::INFO,
+          color: options[:color]
+        }
+        @logger = Fluent::Auditify::Log.new(log_options)
+      end
+
+      def run(options={})
+        if options[:mask_only]
+          @manager = Fluent::Auditify::PluginManager.new(@logger, mask_only: true)
+          @manager.dispatch(options)
+        else
+          @manager = Fluent::Auditify::PluginManager.new(@logger)
+          @manager.dispatch(options)
+          # instance
+          @manager.report(:console)
+        end
+        true
+      end
+    end
+  end
+end

data/lib/fluent/auditify/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Fluent
+  module Auditify
+    VERSION = "0.1.0"
+  end
+end

data/lib/fluent/auditify.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative "auditify/version"
+
+module Fluent
+  module Auditify
+    class DuplicatedPluginError < StandardError; end
+  end
+end

data/sig/fluent/auditify.rbs

@@ -0,0 +1,6 @@
+module Fluent
+  module Auditify
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: fluent-auditify
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kentaro Hayashi
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: parslet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: pastel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: diff-lcs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.2
+- !ruby/object:Gem::Dependency
+  name: term-ansicolor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.11.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.11.2
+description: Audit your Fluentd configuration and raise attention to you
+email:
+- kenhys@gmail.com
+executables:
+- fluent-auditify
+extensions: []
+extra_rdoc_files: []
+files:
+- ".env"
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- exe/fluent-auditify
+- lib/fluent/auditify.rb
+- lib/fluent/auditify/command/auditify.rb
+- lib/fluent/auditify/helper/test.rb
+- lib/fluent/auditify/log.rb
+- lib/fluent/auditify/parser/v1config.rb
+- lib/fluent/auditify/parsletutil.rb
+- lib/fluent/auditify/plugin.rb
+- lib/fluent/auditify/plugin/base.rb
+- lib/fluent/auditify/plugin/conf.rb
+- lib/fluent/auditify/plugin/conf_buffer_file.rb
+- lib/fluent/auditify/plugin/conf_buffer_file_single.rb
+- lib/fluent/auditify/plugin/conf_buffer_memory.rb
+- lib/fluent/auditify/plugin/conf_filter_grep.rb
+- lib/fluent/auditify/plugin/conf_filter_parser.rb
+- lib/fluent/auditify/plugin/conf_filter_record_transformer.rb
+- lib/fluent/auditify/plugin/conf_filter_stdout.rb
+- lib/fluent/auditify/plugin/conf_in_exec.rb
+- lib/fluent/auditify/plugin/conf_in_forward.rb
+- lib/fluent/auditify/plugin/conf_in_http.rb
+- lib/fluent/auditify/plugin/conf_in_monitor_agent.rb
+- lib/fluent/auditify/plugin/conf_in_sample.rb
+- lib/fluent/auditify/plugin/conf_in_syslog.rb
+- lib/fluent/auditify/plugin/conf_in_tail.rb
+- lib/fluent/auditify/plugin/conf_in_tcp.rb
+- lib/fluent/auditify/plugin/conf_in_udp.rb
+- lib/fluent/auditify/plugin/conf_in_unix.rb
+- lib/fluent/auditify/plugin/conf_mask_secrets.rb
+- lib/fluent/auditify/plugin/conf_out_buffer.rb
+- lib/fluent/auditify/plugin/conf_out_copy.rb
+- lib/fluent/auditify/plugin/conf_out_exec.rb
+- lib/fluent/auditify/plugin/conf_out_exec_filter.rb
+- lib/fluent/auditify/plugin/conf_out_file.rb
+- lib/fluent/auditify/plugin/conf_out_forward.rb
+- lib/fluent/auditify/plugin/conf_out_http.rb
+- lib/fluent/auditify/plugin/conf_out_null.rb
+- lib/fluent/auditify/plugin/conf_out_relabel.rb
+- lib/fluent/auditify/plugin/conf_out_rewrite_tag_filter.rb
+- lib/fluent/auditify/plugin/conf_out_roundrobin.rb
+- lib/fluent/auditify/plugin/conf_out_secondary_file.rb
+- lib/fluent/auditify/plugin/conf_out_stdout.rb
+- lib/fluent/auditify/plugin/conf_plugin_params.rb
+- lib/fluent/auditify/plugin/conf_plugin_type.rb
+- lib/fluent/auditify/plugin/conf_v1dupid.rb
+- lib/fluent/auditify/plugin_manager.rb
+- lib/fluent/auditify/registry.rb
+- lib/fluent/auditify/reporter.rb
+- lib/fluent/auditify/reporter/console.rb
+- lib/fluent/auditify/reporter/json.rb
+- lib/fluent/auditify/syntax_checker.rb
+- lib/fluent/auditify/version.rb
+- sig/fluent/auditify.rbs
+homepage: https://github.com/kenhys/fluent-auditify
+licenses:
+- Apache-2.0
+metadata:
+  homepage_uri: https://github.com/kenhys/fluent-auditify
+  source_code_uri: https://github.com/kenhys/fluent-auditify
+  changelog_uri: https://github.com/kenhys/fluent-auditify/blob/master/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Audit tool for your Fluentd configuration
+test_files: []