fluentd-ui 1.0.0.alpha.2 → 1.0.0.alpha.3

data/app/models/fluentd/setting/in_syslog.rb

@@ -1,32 +1,34 @@
 class Fluentd
   module Setting
     class InSyslog
-      include ActiveModel::Model
-      include Common
+      include Fluentd::Setting::Plugin
 
-      KEYS = [
-        :port, :bind, :tag, :types
-      ].freeze
-
-      attr_accessor(*KEYS)
-
-      validates :tag, presence: true
+      register_plugin("input", "syslog")
 
       def self.initial_params
         {
           bind: "0.0.0.0",
           port: 5140,
+          parse_type: "syslog",
+          parse: {
+            "0" => {
+              "type" => "syslog"
+            }
+          },
+          protocol_type: :udp,
         }
       end
 
       def common_options
         [
-          :tag, :bind, :port, :types,
+          :tag, :bind, :port
         ]
       end
 
-      def advanced_options
-        []
+      def hidden_options
+        [
+          :parse
+        ]
       end
     end
   end

data/app/models/fluentd/setting/in_tail.rb

@@ -1,38 +1,16 @@
+require "fluent/plugin/parser_multiline"
+
 class Fluentd
   module Setting
     class InTail
-      MULTI_LINE_MAX_FORMAT_COUNT = 20
-
-      include ActiveModel::Model
-      attr_accessor :path, :tag, :format, :regexp, :time_format, :rotate_wait, :pos_file, :read_from_head, :refresh_interval
-
-      validates :path, presence: true
-      validates :tag, presence: true
-      #validates :format, presence: true
+      include Fluentd::Setting::Plugin
 
-      def self.known_formats
-        {
-          :apache2 => [:time_format],
-          :nginx => [:time_format],
-          :syslog => [:time_format],
-          :tsv => [:keys, :time_key],
-          :csv => [:keys, :time_key],
-          :ltsv => [:delimiter, :time_key],
-          :json => [:time_key],
-          :regexp => [:time_format, :regexp],
-          :multiline => [:format_firstline] + (1..MULTI_LINE_MAX_FORMAT_COUNT).map{|n| "format#{n}".to_sym }
-          # TODO: Grok could generate Regexp including \d, \s, etc. fluentd config parser raise error with them for escape sequence check.
-          #       TBD How to handle Grok/Regexp later, just comment out for hide
-          # :grok => [:grok_str],
-        }
-      end
-      attr_accessor *known_formats.values.flatten.compact.uniq
+      register_plugin("input", "tail")
+      # TODO support formatN ???
 
-      def known_formats
-        self.class.known_formats
-      end
+      MULTI_LINE_MAX_FORMAT_COUNT = ::Fluent::Plugin::MultilineParser::FORMAT_MAX_NUM
 
-      def guess_format
+      def guess_parse_type
         case path
         when /\.json$/
           :json
@@ -53,43 +31,6 @@ class Fluentd
         end
       end
 
-      def extra_format_options
-        self.class.known_formats[format.to_sym] || []
-      end
-
-      def format_specific_conf
-        return "" if %w(grok regexp).include?(format)
-
-        indent = " " * 2
-        format_specific_conf = ""
-
-        if format.to_sym == :multiline
-          known_formats[:multiline].each do |key|
-            value = send(key)
-            if value.present?
-              format_specific_conf << "#{indent}#{key} /#{value}/\n"
-            end
-          end
-        else
-          extra_format_options.each do |key|
-            format_specific_conf << "#{indent}#{key} #{send(key)}\n"
-          end
-        end
-
-        format_specific_conf
-      end
-
-      def certain_format_line
-        case format
-        when "grok"
-          "format /#{grok.convert_to_regexp(grok_str).source.gsub("/", "\\/")}/ # grok: '#{grok_str}'"
-        when "regexp"
-          "format /#{regexp}/"
-        else
-          "format #{format}"
-        end
-      end
-
       def grok
         @grok ||=
           begin
@@ -98,24 +39,6 @@ class Fluentd
             grok
           end
       end
-
-      def to_conf
-        # NOTE: Using strip_heredoc makes more complex for format_specific_conf indent
-        <<-CONFIG.gsub(/^[ ]*\n/m, "")
-<source>
-  type tail
-  path #{path}
-  tag #{tag}
-  #{certain_format_line}
-#{format_specific_conf}
-
-  #{read_from_head.to_i.zero? ? "" : "read_from_head true"}
-  #{pos_file.present? ? "pos_file #{pos_file}" : ""}
-  #{rotate_wait.present? ? "rotate_wait #{rotate_wait}" : ""}
-  #{refresh_interval.present? ? "refresh_interval #{refresh_interval}" : ""}
-</source>
-        CONFIG
-      end
     end
   end
 end

data/app/models/fluentd/setting/out_elasticsearch.rb

@@ -1,24 +1,9 @@
 class Fluentd
   module Setting
     class OutElasticsearch
-      include Common
+      include Fluentd::Setting::Plugin
 
-      KEYS = [
-        :match,
-        :host, :port, :index_name, :type_name,
-        :logstash_format, :logstash_prefix, :logstash_dateformat, :utc_index,
-        :hosts, :request_timeout, :include_tag_key
-      ].freeze
-
-      attr_accessor(*KEYS)
-
-      booleans :logstash_format, :utc_index, :include_tag_key
-
-      validates :match, presence: true
-      validates :host, presence: true
-      validates :port, presence: true
-      validates :index_name, presence: true
-      validates :type_name, presence: true
+      register_plugin("output", "elasticsearch")
 
       def self.initial_params
         {
@@ -29,20 +14,26 @@ class Fluentd
           logstash_format: true,
           include_tag_key: false,
           utc_index: true,
+          buffer_type: "file",
+          buffer: {
+            "0" => {
+              "type" => "file",
+              "path" => "/var/log/td-agent/buffer/elasticsearch",
+             }
+          },
         }
       end
 
       def common_options
         [
-          :match, :host, :port, :logstash_format,
+          :pattern, :host, :port, :logstash_format,
           :index_name, :type_name,
         ]
       end
 
-      def advanced_options
+      def hidden_options
         [
-          :hosts, :logstash_prefix, :logstash_dateformat,
-          :utc_index, :request_timeout, :include_tag_key,
+          :secondary, :inject, :buffer
         ]
       end
     end

data/app/models/fluentd/setting/out_forward.rb

@@ -1,91 +1,47 @@
 class Fluentd
   module Setting
     class OutForward
-      class Server
-        include Common
-        KEYS = [
-          :name, :host, :port, :weight, :standby
-        ].freeze
+      include Fluentd::Setting::Plugin
 
-        attr_accessor(*KEYS)
+      register_plugin("output", "forward")
 
-        flags :standby
-
-        validates :host, presence: true
-        validates :port, presence: true
-      end
-
-      class Secondary
-        include Common
-        KEYS = [
-          :type, :path
-        ].freeze
-
-        attr_accessor(*KEYS)
-
-        hidden :type
-        validates :path, presence: true
-      end
-
-      include Common
-
-      KEYS = [
-        :match,
-        :send_timeout, :recover_wait, :heartbeat_type, :heartbeat_interval,
-        :phi_threshold, :hard_timeout,
-        :server, :secondary
-      ].freeze
-
-      attr_accessor(*KEYS)
-      choice :heartbeat_type, %w(udp tcp)
-      nested :server, Server, multiple: true
-      nested :secondary, Secondary
-
-      validates :match, presence: true
-      validate :validate_has_at_least_one_server
-      validate :validate_nested_values
-
-      def validate_has_at_least_one_server
-        if children_of(:server).reject{|s| s.empty_value? }.blank?
-          errors.add(:base, :out_forward_blank_server)
-        end
-      end
-
-      def validate_nested_values
-        self.class.children.inject(true) do |result, (key, _)|
-          children_of(key).each do |child|
-            if !child.empty_value? && !child.valid?
-              child.errors.full_messages.each do |message|
-                errors.add(:base, "(#{key})#{message}")
-              end
-              result = false
-            end
-            result
-          end
-          result
-        end
+      config_section :secondary do
+        config_param :path, :string
       end
 
       def self.initial_params
         {
+          buffer_type: "memory",
+          buffer: {
+            "0" => {
+              "type" => "memory",
+            }
+          },
           secondary: {
             "0" => {
-              type: "file",
+              "type" => "file",
             }
           }
         }
       end
 
+      # TODO overwrite this method to support transport parameter and transport section
+      # def self.permit_params
+      #   super
+      # end
+
       def common_options
         [
-          :match, :server, :secondary,
+          :pattern, :server, :secondary,
         ]
       end
 
-      def advanced_options
+      def hidden_options
         [
-          :send_timeout, :recover_wait, :heartbeat_type, :heartbeat_interval,
-          :phi_threshold, :hard_timeout,
+          :inject, :buffer,
+          :host, :port,
+          # We don't support TLS configuration via fluentd-ui for now.
+          :transport, :tls_version, :tls_ciphers, :tls_insecure_mode, :tls_verify_hostname, :tls_cert_path
         ]
       end
     end

data/app/models/fluentd/setting/out_mongo.rb

@@ -1,34 +1,20 @@
 class Fluentd
   module Setting
     class OutMongo
-      include Common
+      include Fluentd::Setting::Plugin
 
-      KEYS = [
-        :match,
-        :host, :port, :database, :collection, :capped, :capped_size, :capped_max, :user, :password, :tag_mapped,
-        :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval,  :retry_wait, :retry_limit, :max_retry_wait, :num_threads,
-      ].freeze
+      register_plugin("output", "mongo")
+      config_param(:capped, :bool, default: false)
+      config_param(:capped_size, :size, default: nil)
 
-      attr_accessor(*KEYS)
-
-      flags :capped, :tag_mapped
-
-      validates :match, presence: true
-      validates :host, presence: true
-      validates :port, presence: true
+      # NOTE: fluent-plugin-mongo defines database parameter as required parameter
+      #       But Fluentd tells us that the database parameter is not required.
       validates :database, presence: true
-      validate :validate_capped
       validate :validate_collection
-      validates :buffer_path, presence: true, if: ->{ buffer_type == "file" }
-
-      def validate_capped
-        return true if capped.blank?
-        errors.add(:capped_size, :blank) if capped_size.blank?
-      end
 
       def validate_collection
         if tag_mapped.blank? && collection.blank?
-          errors.add(:collection, :blank) 
+          errors.add(:collection, :blank)
         end
       end
 
@@ -38,20 +24,28 @@ class Fluentd
           port: 27017,
           capped: true,
           capped_size: "100m",
-        }
+          buffer_type: "file",
+          buffer: {
+            "0" => {
+              "type" => "file",
+              "path" => "/var/log/td-agent/buffer/mongo",
+            }
+          },
+         }
       end
 
       def common_options
         [
-          :match, :host, :port, :database, :collection,
+          :pattern, :host, :port, :database, :collection,
           :tag_mapped, :user, :password,
         ]
       end
 
-      def advanced_options
+      def hidden_options
         [
-          :capped, :capped_size, :capped_max, :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit,
-          :flush_interval, :retry_wait, :retry_limit, :max_retry_wait, :num_threads,
+          :secondary, :inject, :buffer,
+          :include_tag_key,
+          :include_time_key
         ]
       end
     end

data/app/models/fluentd/setting/out_s3.rb

@@ -1,51 +1,39 @@
 class Fluentd
   module Setting
     class OutS3
-      include ActiveModel::Model
-      include Common
+      include Fluentd::Setting::Plugin
 
-      KEYS = [
-        :match,
-        :aws_key_id, :aws_sec_key, :s3_bucket, :s3_region, :path,
-        # :reduced_redundancy, :check_apikey_on_start, :command_parameter, # not configurable?
-        :format, :include_time_key, :time_key, :delimiter, :label_delimiter,
-        :time_slice_format, :time_slice_wait, :time_format, :utc, :store_as, :proxy_uri, :use_ssl,
-        :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval,
-        :retry_wait, :retry_limit, :max_retry_wait, :num_threads,
-      ].freeze
-
-      attr_accessor(*KEYS)
-
-      choice :format, %w(out_file json ltsv single_value)
-      choice :store_as, %w(gzip lzo lzma2 json txt)
-      choice :buffer_type, %w(memory file)
-      booleans :include_time_key, :use_ssl
-      flags :utc
-
-      validates :match, presence: true
-      validates :s3_bucket, presence: true
-      validates :buffer_path, presence: true, if: ->{ buffer_type == "file" }
+      register_plugin("output", "s3")
 
       def self.initial_params
         {
           s3_region: "us-west-1",
-          use_ssl: true,
+          buffer_type: "file",
+          buffer: {
+            "0" => {
+              "type" => "file",
+              "path" => "/var/log/td-agent/buffer/s3",
+            }
+          },
+          format_type: "out_file",
+          format: {
+            "0" => {
+              "type" => "out_file"
+            }
+          }
         }
       end
 
       def common_options
         [
-          :match, :aws_key_id, :aws_sec_key,
+          :pattern, :aws_key_id, :aws_sec_key,
           :s3_region, :s3_bucket, :use_ssl, :path,
         ]
       end
 
-      def advanced_options
+      def hidden
         [
-          :format, :include_time_key, :time_key, :delimiter, :label_delimiter,
-          :utc, :time_slice_format, :time_slice_wait, :store_as, :proxy_uri,
-          :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval,
-          :retry_wait, :retry_limit, :max_retry_wait, :num_threads,
+          :secondary, :inject, :buffer
         ]
       end
     end