fluent-auditify 0.1.0

data/lib/fluent/auditify/parser/v1config.rb

@@ -0,0 +1,172 @@
+require 'parslet'
+
+module Fluent
+  module Auditify
+    module Parser
+      class V1ConfigBaseParser < ::Parslet::Parser
+        rule(:space)  { match('[ \t]').repeat(1) }
+        rule(:space?) { space.maybe }
+        rule(:newline) { str("\r\n") | str("\n") | str("\r") }
+        rule(:newline?) { newline.maybe }
+        rule(:integer) { match('[0-9]').repeat(1) }
+        rule(:string) { str('"') >> match('[^"]').repeat >> str('"') }
+        rule(:identifier) { match('[A-Za-z0-9_-]').repeat(1) }
+        rule(:pattern) { match("[A-Za-z0-9_.*{},#'\"\\[\\]]").repeat(1) }
+        rule(:pattern?) { pattern.maybe }
+        rule(:empty_line) { space? >> newline }
+        rule(:comment) { space? >> str('#') >> match('[^\r\n]').repeat >> newline }
+        rule(:space_or_newline) { (space | newline).repeat(1) }
+
+        rule(:eof?) { (newline | any.absent?).maybe }
+        rule(:key) { str('@').maybe >> match('[a-zA-Z0-9_]').repeat(1) }
+        rule(:path) { match('[.a-z0-9A-Z_/\\*\\-]').repeat(1) }
+        rule(:ipv4) {
+          match('[0-9]').repeat(1,3) >>
+            (str('.') >> match('[0-9]').repeat(1,3)).repeat(3) }
+        rule(:nonspace_nonquote_char) { match('[^" \t\r\n]') }
+        rule(:unquoted_word) { (str('"').absent? >> nonspace_nonquote_char.repeat(1)) }
+        rule(:value) { string | unquoted_word }
+        rule(:key_value) { space? >> key.as(:name) >> space >> value.as(:value) >>
+                           space? >> newline }
+        rule(:key_line) { space? >> key.as(:name) >> space_or_newline }
+
+        rule(:tag_name) { match('[a-zA-Z0-9_]').repeat(1) }
+        rule(:open_tag) { str('<') >> tag_name.as(:name) >>
+                          (space >> tag_name.as(:section_arg)).maybe >> str('>') }
+        rule(:close_tag) { str('</') >> tag_name.as(:name) >> str('>') }
+
+        rule(:conf_path) { (match("[^\s\.]+").repeat(1) >> str('.conf')) }
+        rule(:yaml_path) do
+          (match("[^\s\.]+").repeat(1) >> str('.yaml')) |
+            (match("[^\s\.]+").repeat(1) >> str('.yml'))
+        end
+      end
+
+      class V1ConfigParamParser < V1ConfigBaseParser
+        # @include key-value-pair conf
+
+        rule(:conf) { (comment | key_value | key.as(:name) | empty_line).repeat.as(:body) }
+
+        root :conf
+      end
+
+      class V1ConfigSectionParser < V1ConfigBaseParser
+        # @include section conf
+        rule(:tag_name) { match('[a-zA-Z0-9_]').repeat(1) }
+        rule(:open_tag) { str('<') >> tag_name.as(:name) >>
+                          (space >> tag_name.as(:section_arg)).maybe >> str('>') }
+        rule(:close_tag) { str('</') >> tag_name.as(:name) >> str('>') }
+        rule(:section) do
+          space? >> open_tag.as(:section) >> space_or_newline >>
+            (comment | key_value | section | key.as(:name) | empty_line.as(:empty_line)).repeat.as(:body) >>
+            space? >> close_tag >> space? >> eof?
+        end
+
+        rule(:conf) { (comment | key_value | key.as(:name) | empty_line.as(:empty_line) | section).repeat }
+        root :conf
+      end
+
+      class V1ConfigParser < V1ConfigBaseParser
+
+        rule(:system) do
+          space? >> str('<system>').as(:system) >> space_or_newline.maybe >>
+            (comment | key_value | empty_line.as(:empty_line) | section).repeat.as(:body) >>
+            space? >> str('</system>') >> space? >> eof?
+        end
+        rule(:include_directive) do
+          space? >> str('@include').as(:include) >> space >>
+            (conf_path | yaml_path).as(:include_path) >> eof?
+        end
+        rule(:source) do
+          space? >> str('<source>').as(:source) >> space_or_newline >>
+            (comment | empty_line.as(:empty_line) | key_value | key_line | section).repeat.as(:body) >>
+            space? >> str('</source>') >> space_or_newline.maybe >> eof?
+        end
+        rule(:section) do
+          space? >> open_tag.as(:section) >> space_or_newline >>
+            (comment | key_value | empty_line | key_line | section).repeat.as(:body) >>
+            space? >> close_tag >> eof?
+        end
+        rule(:filter) do
+          space? >> str('<filter').as(:filter) >> space? >> pattern?.as(:pattern) >> str('>') >> space_or_newline >>
+            (comment | key_value | empty_line | section).repeat.as(:body) >>
+            space? >> str('</filter>') >> eof?
+        end
+        # match is reserved word
+        rule(:match_directive) do
+          space? >> str('<match').as(:match) >> space? >> pattern?.as(:pattern) >> str('>') >> space_or_newline >>
+            (comment | key_value | empty_line.as(:empty_line) | section).repeat.as(:body) >>
+            space? >> str('</match>') >> eof?
+        end
+        rule(:label) do
+          space? >> str('<label').as(:label) >> space? >> key.as(:name) >> str('>') >> space_or_newline >>
+            (comment | filter | match_directive).repeat.as(:body) >>
+            space? >> str('</label>') >> eof?
+        end
+        rule(:directive) { system | source | filter | match_directive | include_directive | label} # | filter | match | label | empty_line }
+        rule(:conf) { (directive | comment | empty_line.as(:empty_line)).repeat(1) }
+
+        root :conf
+
+        # expand @include directive
+        def self.eval(object, base_dir: "", path: "", include: true)
+          modified = []
+          object.each_with_index do |element, index|
+            element[:__BASE__] = base_dir
+            element[:__PATH__] = path
+            unless element[:include]
+              if element[:empty_line]
+                modified << element
+              elsif element[:body].collect { |v| v[:name].to_s }.any?('@include')
+                # include section
+                modified_body = []
+                element[:body].each do |body_element|
+                  if body_element[:name].to_s == '@include'
+                    parser = Fluent::Auditify::Parser::V1ConfigSectionParser.new
+                    parsed = parser.parse(File.read(File.join(base_dir, body_element[:value])))
+                    parsed.each do |elem|
+                      elem[:__PATH__] = body_element[:value].to_s
+                      modified_body << elem
+                    end
+                  else
+                    modified_body << body_element
+                  end
+                end
+                element[:body] = modified_body
+                modified << element
+              else
+                modified << element
+              end
+              next
+            end
+            parser = Fluent::Auditify::Parser::V1ConfigParser.new
+            pattern = File.expand_path(element[:include_path].to_s, base_dir)
+            Dir.glob(pattern).sort.each do |path|
+              included =  parser.parse(File.read(path))
+              included.each do |child|
+                child[:__PATTERN__] = element[:include_path].to_s
+                child[:__PATH__] = Pathname.new(path).relative_path_from(base_dir).to_s
+                child[:__BASE__] = base_dir
+                modified << child
+              end
+            end
+          end
+          modified
+        end
+
+        def find_nth_element(object, nth: 1, elements: [])
+          count = 0
+          elements.each do |element|
+            if element[object.intern]
+              count += 1
+              if nth == count
+                return element
+              end
+            end
+          end
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/fluent/auditify/parsletutil.rb

@@ -0,0 +1,159 @@
+require 'stringio'
+
+module Fluent
+  module Auditify
+    class ParsletUtil
+      def initialize(options={})
+        reset_style
+      end
+
+      def reset_style
+        @indent_level = 0
+        @align = 2
+        @content = StringIO.new
+      end
+
+      def collect_file_handlers(object)
+        handlers = {}
+        object.each do |directive|
+          if directive[:empty_line]
+            key = directive[:__PATH__]
+            unless key and handlers.key?(key)
+              if key
+                handlers[key] = File.open(key, 'w+')
+              end
+            end
+          elsif directive[:source] or directive[:match] or
+               directive[:system] or directive[:filter]
+            key = directive[:__PATH__]
+            unless key and handlers.key?(key)
+              if key
+                handlers[key] = File.open(key, 'w+')
+              end
+            end
+            directive[:body].each do |body|
+              key = body[:__PATH__]
+              unless key and handlers.key?(key)
+                if key
+                  handlers[key] = File.open(key, 'w+')
+                end
+              end
+            end
+          end
+        end
+        handlers
+      end
+
+      def export(object, options={})
+        # setup rewrite file handles
+        @handlers = collect_file_handlers(object)
+        object.each do |directive|
+          io = @handlers[directive[:__PATH__]]
+          if directive[:system]
+            io.puts("#{' ' * @align * @indent_level}#{directive[:system].to_s}") if io
+            export_body(directive)
+            io.puts('</system>') if io
+          elsif directive[:source]
+            io.puts "#{' ' * @align * @indent_level}#{directive[:source].to_s}" if io
+            export_body(directive)
+            io.puts('</source>') if io
+          elsif directive[:filter]
+            io.puts "#{' ' * @align * @indent_level}#{directive[:filter].to_s}" if io
+            export_body(directive)
+            io.puts('</filter>') if io
+          elsif directive[:match]
+            io.puts "#{' ' * @align * @indent_level}#{directive[:match].to_s}"
+            export_body(directive)
+            io.puts('</match>') if io
+          end
+        end
+      end
+
+      def export_body(directive)
+        directive[:body].each do |child|
+          if child[:section]
+          elsif child[:empty_line]
+            if child[:__PATH__]
+              io = @handlers[child[:__PATH__]]
+              if io
+                io.puts
+              end
+            else
+              io = @handlers[directive[:__PATH__]]
+              if io
+                io.puts
+              end
+            end
+          elsif child[:value]
+            io = @handlers[child[:__PATH__]]
+            if io
+              io.puts("#{' ' * @align * @indent_level}#{child[:name].to_s} #{child[:value].to_s}")
+            end
+          elsif child[:name]
+            io = @handlers[child[:__PATH__]]
+            if io
+              io.puts("#{' ' * @align * @indent_level}#{child[:name].to_s}")
+            end
+          end
+        end
+      end
+
+      def to_s(object, options={})
+        object.each do |directive|
+          if directive[:source]
+            @content.puts "#{' ' * @align * @indent_level}#{directive[:source].to_s}"
+            stringify_body(directive)
+            @content.puts "</source>"
+          elsif directive[:match]
+            @content.puts "#{' ' * @align * @indent_level}#{directive[:match].to_s}>"
+            stringify_body(directive)
+            @content.puts "</match>"
+          elsif directive[:system]
+            @content.puts "#{' ' * @align * @indent_level}#{directive[:system].to_s}"
+            stringify_body(directive)
+            @content.puts "</system>"
+          elsif directive[:empty_line]
+            @content.puts
+          else
+          end
+        rescue => e
+          p e
+        end
+        @content.string
+      end
+
+      private
+
+      def stringify_body(directive)
+        @indent_level += 1
+        directive[:body].each do |child|
+          if child[:section]
+            stringify_section(child)
+          elsif child[:empty_line]
+            @content.puts
+          elsif child[:value]
+            @content.puts "#{' ' * @align * @indent_level}#{child[:name].to_s} #{child[:value].to_s}"
+          elsif child[:name]
+            @content.puts "#{' ' * @align * @indent_level}#{child[:name].to_s}"
+          else
+          end
+        end
+        @indent_level -= 1
+      end
+
+      def stringify_section(section)
+        @content.puts "#{' ' * @align * @indent_level}<#{section[:section][:name].to_s}>"
+        @indent_level += 1
+        section[:body].each do |child|
+          if child[:section]
+            stringify_section(child)
+          elsif child[:name]
+            @content.puts "#{' ' * @align * @indent_level}#{child[:name].to_s} #{child[:value].to_s}"
+          end
+        end
+        @indent_level -= 1
+        @content.puts "#{' ' * @align * @indent_level}</#{section[:name].to_s}>"
+      end
+    end
+  end
+end

data/lib/fluent/auditify/plugin/base.rb

@@ -0,0 +1,17 @@
+require 'fluent/auditify/plugin'
+
+module Fluent
+  module Auditify
+    module Plugin
+      class Base
+        attr_reader :log
+        def initialize
+          @disabled = false
+          @options = {
+            config_version: :v1
+          }
+        end
+      end
+    end
+  end
+end

data/lib/fluent/auditify/plugin/conf.rb

@@ -0,0 +1,135 @@
+require 'fluent/auditify/log'
+require 'fluent/auditify/plugin/base'
+
+module Fluent
+  module Auditify
+    module Plugin
+      class Conf < Base
+
+        def initialize
+          super
+        end
+
+        def supported_platform?
+          raise NotImplementedError
+        end
+
+        def supported_file_extension?
+          [:conf]
+        end
+
+        def disabled?
+          @disabled
+        end
+
+        def parse(config_path, options={})
+          raise NotImplementedError
+        end
+
+        def yaml?(path)
+          path.end_with?('.yml', '.yaml')
+        end
+
+        def conf?(path)
+          path.end_with?('.conf')
+        end
+
+        def read_with_include_directive(path)
+          contents = []
+          File.open(path) do |f|
+            f.readlines.each_with_index do |line, index|
+              if line.strip.start_with?('@include')
+                target = File.expand_path(line.split.last, File.dirname(path))
+                contents = (contents << file_get_contents(target, lines: true, include: true)).flattern
+              else
+                contents << {line: index, content: line, path: path}
+              end
+            end
+          end
+          contents
+        end
+
+        def file_get_contents(path, lines: false, include: false)
+          contents = []
+          if lines
+            if include
+              contents = read_with_include_directive(path)
+            else
+              File.open(path) do |f|
+                f.readlines.each_with_index do |line, index|
+                  contents << {line: index, content: line, path: path}
+                end
+              end
+            end
+          else
+            if include
+              contents = read_with_include_directive(path).collect do |entry|
+                entry[:content]
+              end.join
+            else
+              contents = File.open(path) do |f| f.read end
+            end
+          end
+          contents
+        end
+
+        def file_readlines_each(conf)
+          File.open(conf) do |f|
+            f.readlines.each_with_index do |line, index|
+              yield line, index
+            end
+          end
+        end
+
+        def guilty(level, message, options={})
+          Plugin.guilty(level, message, options)
+        end
+
+        def polish(object)
+          Plugin.polish(object)
+        end
+
+        def artifact
+          Plugin.artifact
+        end
+
+        def plugin_defs(type, plugin_name)
+          spec = {}
+          begin
+            IO.popen(['fluent-plugin-config-format', '--compact', '--format', 'json', type, plugin_name]) do |io|
+              json = JSON.parse(io.read)
+              json.each do |klass, defs|
+                next if klass == 'plugin_helpers'
+                next if klass == "Fluent::Plugin::#{type[0].upcase}#{type[1..]}"
+                next if klass.split('::').count != 3
+                spec = defs
+              end
+            end
+          rescue => e
+            log.error("failed to get plugin specification: #{e.message}")
+          end
+          spec
+        end
+
+        def surround_text(path, line_num, range: 2, replace: nil)
+          lines = file_get_contents(path)
+          min = line_num - range > 0 ? line - 2 : 0
+          max = line_num + range < lines.size ? line_num + range : lines.size - 1
+          content = ""
+          min.upto(max).each_with_index do |line, index|
+            if replace
+              if min + index + 1 == line_num
+                content << "#{replace}\n"
+              else
+                content << "#{min + index + 1}: #{lines[min + index].chomp}\n"
+              end
+            else
+              content << "#{min + index + 1}: #{lines[min + index].chomp}\n"
+            end
+          end
+          content
+        end
+      end
+    end
+  end
+end

data/lib/fluent/auditify/plugin/conf_buffer_file.rb

@@ -0,0 +1,18 @@
+require 'fluent/config/error'
+require 'fluent/auditify/plugin/conf'
+
+module Fluent::Auditify::Plugin
+  class ConfBufferFile < Conf
+    Fluent::Auditify::Plugin.register_conf('buffer_file', self)
+
+    def supported_platform?
+      :any
+    end
+
+    def initialize
+    end
+
+    def parse(conf, options={})
+    end
+  end
+end

data/lib/fluent/auditify/plugin/conf_buffer_file_single.rb

@@ -0,0 +1,18 @@
+require 'fluent/config/error'
+require 'fluent/auditify/plugin/conf'
+
+module Fluent::Auditify::Plugin
+  class ConfBufferFileSingle < Conf
+    Fluent::Auditify::Plugin.register_conf('buffer_file_single', self)
+
+    def supported_platform?
+      :any
+    end
+
+    def initialize
+    end
+
+    def parse(conf, options={})
+    end
+  end
+end

data/lib/fluent/auditify/plugin/conf_buffer_memory.rb

@@ -0,0 +1,18 @@
+require 'fluent/config/error'
+require 'fluent/auditify/plugin/conf'
+
+module Fluent::Auditify::Plugin
+  class ConfBufferMemory < Conf
+    Fluent::Auditify::Plugin.register_conf('buffer_memory', self)
+
+    def supported_platform?
+      :any
+    end
+
+    def initialize
+    end
+
+    def parse(conf, options={})
+    end
+  end
+end

data/lib/fluent/auditify/plugin/conf_filter_grep.rb

@@ -0,0 +1,18 @@
+require 'fluent/config/error'
+require 'fluent/auditify/plugin/conf'
+
+module Fluent::Auditify::Plugin
+  class ConfFilterGrep < Conf
+    Fluent::Auditify::Plugin.register_conf('filter_grep', self)
+
+    def supported_platform?
+      :any
+    end
+
+    def initialize
+    end
+
+    def parse(conf, options={})
+    end
+  end
+end

data/lib/fluent/auditify/plugin/conf_filter_parser.rb

@@ -0,0 +1,18 @@
+require 'fluent/config/error'
+require 'fluent/auditify/plugin/conf'
+
+module Fluent::Auditify::Plugin
+  class ConfFilterParser < Conf
+    Fluent::Auditify::Plugin.register_conf('filter_parser', self)
+
+    def supported_platform?
+      :any
+    end
+
+    def initialize
+    end
+
+    def parse(conf, options={})
+    end
+  end
+end

data/lib/fluent/auditify/plugin/conf_filter_record_transformer.rb

@@ -0,0 +1,18 @@
+require 'fluent/config/error'
+require 'fluent/auditify/plugin/conf'
+
+module Fluent::Auditify::Plugin
+  class ConfFilterRecordTransformer < Conf
+    Fluent::Auditify::Plugin.register_conf('filter_record_transformer', self)
+
+    def supported_platform?
+      :any
+    end
+
+    def initialize
+    end
+
+    def parse(conf, options={})
+    end
+  end
+end

data/lib/fluent/auditify/plugin/conf_filter_stdout.rb

@@ -0,0 +1,18 @@
+require 'fluent/config/error'
+require 'fluent/auditify/plugin/conf'
+
+module Fluent::Auditify::Plugin
+  class ConfFilterStdout < Conf
+    Fluent::Auditify::Plugin.register_conf('filter_stdout', self)
+
+    def supported_platform?
+      :any
+    end
+
+    def initialize
+    end
+
+    def parse(conf, options={})
+    end
+  end
+end

data/lib/fluent/auditify/plugin/conf_in_exec.rb

@@ -0,0 +1,18 @@
+require 'fluent/config/error'
+require 'fluent/auditify/plugin/conf'
+
+module Fluent::Auditify::Plugin
+  class ConfInExec < Conf
+    Fluent::Auditify::Plugin.register_conf('in_exec', self)
+
+    def supported_platform?
+      :any
+    end
+
+    def initialize
+    end
+
+    def parse(conf, options={})
+    end
+  end
+end

data/lib/fluent/auditify/plugin/conf_in_forward.rb

@@ -0,0 +1,18 @@
+require 'fluent/config/error'
+require 'fluent/auditify/plugin/conf'
+
+module Fluent::Auditify::Plugin
+  class ConfInForward < Conf
+    Fluent::Auditify::Plugin.register_conf('in_forward', self)
+
+    def supported_platform?
+      :any
+    end
+
+    def initialize
+    end
+
+    def parse(conf, options={})
+    end
+  end
+end

data/lib/fluent/auditify/plugin/conf_in_http.rb

@@ -0,0 +1,18 @@
+require 'fluent/config/error'
+require 'fluent/auditify/plugin/conf'
+
+module Fluent::Auditify::Plugin
+  class ConfInHttp < Conf
+    Fluent::Auditify::Plugin.register_conf('in_http', self)
+
+    def supported_platform?
+      :any
+    end
+
+    def initialize
+    end
+
+    def parse(conf, options={})
+    end
+  end
+end