fluentd-ui 1.0.1 → 1.1.0

data/app/models/concerns/fluentd/setting/configurable.rb

@@ -35,6 +35,7 @@ class Fluentd
         self.class._sections.each do |name, klass|
           klass.init
           if klass.multi
+            next if attributes.nil?
             next if attributes[name].nil?
             attributes[name].each do |attr|
               next unless attr

data/app/models/concerns/fluentd/setting/label.rb

@@ -0,0 +1,11 @@
+class Fluentd
+  module Setting
+    module Label
+      extend ActiveSupport::Concern
+
+      included do
+        config_argument(:label, :string)
+      end
+    end
+  end
+end

data/app/models/concerns/fluentd/setting/plugin.rb

@@ -1,4 +1,5 @@
 require "fluent/plugin"
+require "fluent/plugin/buf_file"
 require "fluent/test/log"
 require "fluent/test/driver/input"
 require "fluent/test/driver/output"
@@ -17,6 +18,7 @@ class Fluentd
       include Fluentd::Setting::PluginConfig
       include Fluentd::Setting::SectionParser
       include Fluentd::Setting::PluginParameter
+      include Fluentd::Setting::Label
 
       included do
         cattr_accessor :plugin_type, :plugin_name, :config_definition

data/app/models/concerns/fluentd/setting/plugin_config.rb

@@ -10,7 +10,12 @@ class Fluentd
       def validate_configuration
         original_log = $log
         $log = DummyLogger.logger
-        config = to_config.to_s.lines[1..-2].join
+        full_config = to_config.to_s
+        config = if full_config.start_with?("<label ")
+                   full_config.lines[2..-3].join
+                 else
+                   full_config.lines[1..-2].join
+                 end
         self.class.create_driver(config)
       rescue Fluent::ConfigError => ex
         errors.add(:base, :invalid, message: ex.message)
@@ -40,6 +45,10 @@ class Fluentd
         end
         _attributes = { "@type" => self.plugin_name }.merge(_attributes)
         _attributes["@log_level"] = _attributes.delete("log_level")
+        label = _attributes.delete("label")
+        if plugin_type == "input"
+          _attributes["@label"] = label
+        end
         argument = case plugin_type
                    when "output", "filter", "buffer"
                      _attributes.delete(self._argument_name.to_s) || ""
@@ -47,7 +56,18 @@ class Fluentd
                      ""
                    end
         attrs, elements = parse_attributes(_attributes)
-        config_element(name, argument, attrs, elements)
+        case plugin_type
+        when "output", "filter"
+          # e is <match> or <filter>
+          e = config_element(name, argument, attrs, elements)
+          if label.blank?
+            e
+          else
+            config_element("label", label, {}, [e])
+          end
+        else
+          config_element(name, argument, attrs, elements)
+        end
       end
 
       def parse_attributes(attributes)
@@ -70,11 +90,21 @@ class Fluentd
               }
             end
           end
-          elements = section_params.map do |index, _section_params|
+          _elements = section_params.map do |index, _section_params|
             section_class.new(_section_params).to_config
           end.compact
+          elements.concat(_elements)
+        end
+        params = params.to_h
+        if plugin_type == "filter" && plugin_name == "record_transformer"
+          record_params = {}
+          params.delete("record").lines.each do |line|
+            k, v = line.split(" ", 2)
+            record_params[k] = v
+          end
+          elements << config_element("record", "", record_params, [])
         end
-        attrs = params.to_h.reject do |key, value|
+        attrs = params.reject do |key, value|
           skip?(key.to_sym, value)
         end
         return attrs, elements

data/app/models/concerns/fluentd/setting/section_config.rb

@@ -22,9 +22,17 @@ class Fluentd
         end
         elements = sections.map do |key, section_params|
           if section_params.present?
-            self._sections[key.to_sym].new(section_params).to_config
+            section_class = self._sections[key.to_sym]
+            if section_class.multi?
+              section_params.map do |index, _section_params|
+                section_class.new(_section_params).to_config
+              end
+            else
+              section_class.new(section_params).to_config
+            end
           end
-        end.compact
+        end
+        elements = elements.flatten.compact
         attrs = params.to_h.reject do |key, value|
           skip?(key.to_sym, value)
         end

data/app/models/fluentd/agent/common.rb

@@ -18,6 +18,8 @@
 #   - https://github.com/treasure-data/omnibus-td-agent/blob/master/templates/etc/systemd/td-agent.service.erb#L14
 #   fluentd: /etc/fluent/fluent.conf (created by fluentd -s)
 
+require "strscan"
+
 class Fluentd
   class Agent
     module Common
@@ -74,6 +76,25 @@ class Fluentd
         end
       end
 
+      def config_merge(content)
+        if content.start_with?("<label ")
+          label = content.slice(/<label\s+(.+?)>/, 1)
+          key = "label:#{label}"
+          parsed_config = parse_config(config)
+          if parsed_config.key?(key)
+            offset = parsed_config[key][0][:pos] + parsed_config[key][0][:size]
+            label, sections = parse_label_section(content, offset)
+            parsed_config[key][0][:sections]["filter"].concat(sections["filter"])
+            parsed_config[key][0][:sections]["match"].concat(sections["match"])
+            config_write(dump_parsed_config(parsed_config))
+          else
+            config_append(content)
+          end
+        else
+          config_append(content)
+        end
+      end
+
       def configuration
         if File.exists? config_file
           ::Fluentd::Agent::Configuration.new(config_file)
@@ -142,6 +163,82 @@ class Fluentd
           FileUtils.rm(file) if File.exist? file
         end
       end
+
+      def parse_config(content)
+        scanner = StringScanner.new(content)
+        contents = Hash.new {|h, k| h[k] = [] }
+        until scanner.eos? do
+          started = scanner.pos
+          header = scanner.scan_until(/^<(source|filter|match|label)/)
+          section_type = scanner[1]
+          break unless header
+          case section_type
+          when "source", "filter", "match"
+            current_source = header + scanner.scan_until(%r{^</(?:source|filter|match)>})
+            contents[section_type] << { pos: started, content: current_source.strip }
+          when "label"
+            label_content = header + scanner.scan_until(%r{^</label>})
+            label, sections = parse_label_section(label_content, started)
+            contents["label:#{label}"] << { label: label, pos: started, sections: sections, size: label_content.size }
+          else
+            raise TypeError, "Unknown section: #{started}: #{section_type}"
+          end
+        end
+        contents
+      end
+
+      def parse_label_section(content, offset)
+        scanner = StringScanner.new(content)
+        scanner.scan_until(/^<label\s+?([^\s]+?)>/)
+        label = scanner[1]
+        sections = Hash.new {|h, k| h[k] = [] }
+        loop do
+          break if scanner.match?(%r{\s+?</label>})
+          pos = scanner.pos
+          header = scanner.scan_until(/^\s*<(filter|match)/)
+          type = scanner[1]
+          source = header + scanner.scan_until(%r{^\s*</(?:filter|match)>})
+          sections[type] << { label: label, pos: pos + offset, content: source.sub(/\n+/, "") }
+        end
+        return label, sections
+      end
+
+      def dump_parsed_config(parsed_config)
+        content = "".dup
+        sources = parsed_config["source"] || []
+        filters = parsed_config["filter"] || []
+        matches = parsed_config["match"] || []
+        labels = parsed_config.select do |key, sections|
+          key.start_with?("label:")
+        end
+        labels = labels.values.flatten
+        sorted_sections = (sources + filters + matches + labels).sort_by do |section|
+          section[:pos]
+        end
+        sorted_sections.each do |section|
+          if section.key?(:label)
+            label = section[:label]
+            sub_filters = section.dig(:sections, "filter") || []
+            sub_matches = section.dig(:sections, "match") || []
+            sorted_sub_filters = sub_filters.sort_by do |sub_section|
+              sub_section[:pos]
+            end
+            sorted_sub_matches =  sub_matches.sort_by do |sub_section|
+              sub_section[:pos]
+            end
+            sub_sections = sorted_sub_filters + sorted_sub_matches
+            content << "<label #{label}>\n"
+            sub_sections.each do |sub_section|
+              content << sub_section[:content] << "\n\n"
+            end
+            content.chomp!
+            content << "</label>\n\n"
+          else
+            content << section[:content] << "\n\n"
+          end
+        end
+        content.chomp
+      end
     end
   end
 end

data/app/models/fluentd/setting/config.rb

@@ -21,12 +21,83 @@ class Fluentd
         end
       end
 
+      def filters
+        elements.find_all do |elm|
+          elm.name == "filter"
+        end
+      end
+
       def matches
         elements.find_all do |elm|
           elm.name == "match"
         end
       end
 
+      def labels
+        elements.find_all do |elm|
+          elm.name == "label"
+        end
+      end
+
+      def group_by_label
+        hash = Hash.new{|h, k| h[k] = {} }
+        sources.each do |source|
+          label = source["@label"] || source["label"]
+          if label
+            hash[label][:sources] = [source]
+          else
+            hash["ROOT"][:sources] = [source]
+          end
+        end
+        hash["ROOT"][:filters] = filters unless filters.empty?
+        hash["ROOT"][:matches] = matches unless matches.empty?
+
+        labels.each do |label|
+          hash[label.arg][:filters] = label.elements.find_all do |e|
+            e.name == "filter"
+          end
+          hash[label.arg][:matches] = label.elements.find_all do |e|
+            e.name == "match"
+          end
+        end
+        hash
+      end
+
+      def delete_element(name, arg, element)
+        if name == "label"
+          label_section = fl_config.elements(name: name, arg: arg).first
+          original_size = label_section.elements.size
+          remaining_elements = label_section.elements.reject do |e|
+            element == e
+          end
+          if remaining_elements.empty?
+            remaining_elements = fl_config.elements.reject do |e|
+              label_section == e
+            end
+            fl_config.elements = remaining_elements
+            return element
+          else
+            label_section.elements = remaining_elements
+            if original_size == label_section.elements.size
+              return nil
+            else
+              return element
+            end
+          end
+        else
+          original_size = fl_config.elements.size
+          remaining_elements = fl_config.elements.reject do |e|
+            element == e
+          end
+          fl_config.elements = remaining_elements
+          if original_size == fl_config.elements.size
+            return nil
+          else
+            return element
+          end
+        end
+      end
+
       def write_to_file
         return unless Fluentd.instance
         Fluentd.instance.agent.config_write formatted

data/app/models/fluentd/setting/filter_grep.rb

@@ -0,0 +1,38 @@
+class Fluentd
+  module Setting
+    class FilterGrep
+      include Fluentd::Setting::Plugin
+
+      register_plugin("filter", "grep")
+
+      def self.initial_params
+        {
+        }
+      end
+
+      def self.permit_params
+        [
+          :label,
+          :pattern, :log_level, :@log_level,
+          { and: {regexp: [:key, :pattern], exclude: [:key, :pattern]} },
+          { or: {regexp: [:key, :pattern], exclude: [:key, :pattern]} }
+        ]
+      end
+
+      def common_options
+        [
+          :label,
+          :pattern,
+        ]
+      end
+
+      def hidden_options
+        regexps = (1..20).map {|n| :"regexp#{n}"}
+        excludes = (1..20).map {|n| :"exclude#{n}"}
+        [
+          *regexps, *excludes, :regexp, :exclude, :and, :or
+        ]
+      end
+    end
+  end
+end

data/app/models/fluentd/setting/filter_parser.rb

@@ -0,0 +1,34 @@
+class Fluentd
+  module Setting
+    class FilterParser
+      include Fluentd::Setting::Plugin
+
+      register_plugin("filter", "parser")
+
+      def self.initial_params
+        {
+          parse_type: "none",
+          parse: {
+            "0" => {
+              "type" => "none"
+            }
+          }
+        }
+      end
+
+      def common_options
+        [
+          :label,
+          :pattern,
+          :key_name,
+        ]
+      end
+
+      def hidden_options
+        [
+          :parse
+        ]
+      end
+    end
+  end
+end

data/app/models/fluentd/setting/filter_record_transformer.rb

@@ -0,0 +1,27 @@
+class Fluentd
+  module Setting
+    class FilterRecordTransformer
+      include Fluentd::Setting::Plugin
+
+      register_plugin("filter", "record_transformer")
+
+      attribute(:record, :string)
+
+      def self.initial_params
+        {
+        }
+      end
+
+      def common_options
+        [
+          :label,
+          :pattern,
+        ]
+      end
+
+      def hidden_options
+        []
+      end
+    end
+  end
+end

data/app/models/fluentd/setting/filter_stdout.rb

@@ -0,0 +1,34 @@
+class Fluentd
+  module Setting
+    class FilterStdout
+      include Fluentd::Setting::Plugin
+
+      register_plugin("filter", "stdout")
+
+      def self.initial_params
+        {
+          format_type: "stdout",
+          format: {
+            "0" => {
+              "@type" => "stdout",
+              "output_type" => "json"
+            }
+          }
+        }
+      end
+
+      def common_options
+        [
+          :label,
+          :pattern,
+        ]
+      end
+
+      def hidden_options
+        [
+          :inject
+        ]
+      end
+    end
+  end
+end