fluentd-ui 0.1.0

data/app/models/fluentd/agent/local_common.rb

@@ -0,0 +1,101 @@
+class Fluentd
+  class Agent
+    module LocalCommon
+      def running?
+        begin
+          pid && Process.kill(0, pid)
+        rescue Errno::ESRCH
+          File.unlink(pid_file) # no needed any more
+          false
+        end
+      end
+
+      def log
+        return "" unless File.exists?(log_file)
+        File.read(log_file) # TODO: large log file
+      end
+
+      def config
+        File.read(config_file)
+      end
+
+      def config_write(content)
+        File.open(config_file, "w") do |f|
+          f.write content
+        end
+      end
+
+      def config_append(content)
+        File.open(config_file, "a") do |f|
+          f.write "\n"
+          f.write content
+        end
+      end
+
+      def log_tail(limit = nil)
+        return [] unless File.exists?(log_file)
+
+        limit = limit.to_i rescue 0
+        limit = limit.zero? ? Settings.default_log_tail_count : limit
+        io = File.open(log_file)
+        buf = []
+        reader = ::FileReverseReader.new(io)
+        reader.each_line do |line|
+          buf << line
+          break if buf.length >= limit
+        end
+        buf
+      end
+
+      def configuration
+        if File.exists? config_file
+          ::Fluentd::Agent::Configuration.new(config_file)
+        end
+      end
+
+      private
+
+      def pid
+        return unless File.exists?(pid_file)
+        File.read(pid_file).to_i rescue nil
+      end
+
+      def logged_errors(&block)
+        return [] unless File.exist?(log_file)
+        buf = []
+        io = File.open(log_file)
+        reader = ::FileReverseReader.new(io)
+        reader.each_line do |line|
+          unless line["error"]
+            if buf.present?
+              # NOTE: if a following log is given
+              #         2014-06-30 11:24:08 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=#<Errno::EADDRINUSE: Address already in use - bind(2) for 0.0.0.0:24220>
+              #         2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `bind'
+              #         2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `listen'
+              #         2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:461:in `block in tcp_server_sockets'
+              #       the first line become a "subject", trailing lines are "notes"
+              #       {
+              #         subject: "2014-06-30 11:24:08 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=#<Errno::EADDRINUSE: Address already in use - bind(2) for 0.0.0.0:24220>",
+              #         notes: [
+              #           2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `bind'
+              #           2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `listen'
+              #           2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:461:in `block in tcp_server_sockets'
+              #         ]
+              #       }
+              subject, *notes = *buf.reverse
+              block.call({
+                subject: subject,
+                notes: notes,
+              })
+            end
+            buf = []
+            next
+          end
+          buf << line
+        end
+      ensure
+        io && io.close
+      end
+    end
+  end
+end

data/app/models/fluentd/agent/remote.rb

@@ -0,0 +1,7 @@
+class Fluentd
+  class Agent
+    class Remote # TODO
+      include Common
+    end
+  end
+end

data/app/models/fluentd/agent/td_agent.rb

@@ -0,0 +1,44 @@
+class Fluentd
+  class Agent
+    class TdAgent
+      include Common
+      include LocalCommon
+
+      def self.default_options
+        {
+          :pid_file => "/var/run/td-agent/td-agent.pid",
+          :log_file => "/var/log/td-agent/td-agent.log",
+          :config_file => "/etc/td-agent/td-agent.conf",
+        }
+      end
+
+      def start
+        detached_command('/etc/init.d/td-agent start')
+      end
+
+      def stop
+        detached_command('/etc/init.d/td-agent stop')
+      end
+
+      def restart
+        # NOTE: td-agent has no reload command
+        # https://github.com/treasure-data/td-agent/blob/master/debian/td-agent.init#L156
+        detached_command('/etc/init.d/td-agent restart')
+      end
+
+      def version
+        `/usr/sbin/td-agent --version`.strip
+      end
+
+      private
+
+      def detached_command(cmd)
+        Bundler.with_clean_env do
+          pid = spawn(cmd)
+          Process.detach(pid)
+        end
+        sleep 1 # NOTE/FIXME: too early return will be caused incorrect status report, "sleep 1" is a adhoc hack
+      end
+    end
+  end
+end

data/app/models/fluentd/api.rb

@@ -0,0 +1,6 @@
+require "fluentd/api/http"
+
+class Fluentd
+  class Api
+  end
+end

data/app/models/fluentd/api/http.rb

@@ -0,0 +1,26 @@
+require "httpclient"
+require "addressable/uri"
+
+class Fluentd
+  class Api
+    class Http
+      def initialize(endpoint)
+        @endpoint = Addressable::URI.parse(endpoint)
+      end
+
+      def config
+        request("/api/config.json")
+      end
+
+      private
+
+      def request(path)
+        uri = @endpoint.dup
+        uri.path = path
+        res = HTTPClient.get(uri)
+        JSON.parse res.body
+      end
+    end
+  end
+end
+

data/app/models/fluentd/setting.rb

@@ -0,0 +1,4 @@
+class Fluentd
+  module Setting
+  end
+end

data/app/models/fluentd/setting/common.rb

@@ -0,0 +1,185 @@
+class Fluentd
+  module Setting
+    module Common
+      extend ActiveSupport::Concern
+      include ActiveModel::Model
+
+      module ClassMethods
+        attr_accessor :values, :types, :children, :hidden_values
+
+        def choice(key, values)
+          @values ||= {}
+          @values[key] = values
+          set_type(:choice, [key])
+        end
+
+        def hidden(key)
+          set_type(:hidden, [key])
+        end
+
+        def nested(key, klass, options = {})
+          # e.g.:
+          #  <match>
+          #    type forward
+          #    <server>
+          #      ..
+          #    </server>
+          #  </match>
+          @children ||= {}
+          @children[key] = {
+            class: klass,
+            options: options,
+          }
+          set_type(:nested, [key])
+        end
+
+        def booleans(*keys)
+          # e.g.:
+          #   use_ssl true
+          #   include_time_key false
+          set_type(:boolean, keys)
+        end
+
+        def flags(*keys)
+          # e.g.:
+          #   tag_mapped
+          #   utc
+          set_type(:flag, keys)
+        end
+
+        private
+
+        def set_type(type, keys)
+          @types ||= {}
+          keys.each do |key|
+            @types[key] = type
+          end
+        end
+      end
+
+      def children_of(key)
+        meta = self.class.children[key]
+        return unless meta
+        klass = meta[:class]
+        data = send(key) || {"0" => {}}
+        children = []
+        data.each_pair do |index, attrs|
+          children << klass.new(attrs)
+        end
+        children
+      end
+
+      def child_class(key)
+        self.class.children[key][:class]
+      end
+
+      def values_of(key)
+        self.class.values.try(:[], key) || []
+      end
+
+      def column_type(key)
+        self.class.types.try(:[], key) || "string"
+      end
+
+      def conf(key)
+        case column_type(key)
+        when :boolean
+          boolenan(key)
+        when :flag
+          flag(key)
+        when :nested
+          return "" unless send(key)
+          klass = child_class(key)
+          send(key).map do |(_, child)|
+            # send("servers")
+            #
+            # "servers" => {
+            #   "0" => {
+            #     "name" => "foo",
+            #     "host" => "bar",
+            #     ..
+            #   },
+            #   "1" => {
+            #     ..
+            #   }
+            # }
+            child_instance = klass.new(child)
+            unless child_instance.empty_value?
+              "\n" + child_instance.to_config(key).gsub(/^/m, "  ")
+            end
+          end.join
+        else # including :hidden
+          print_if_present(key)
+        end
+      end
+
+      def plugin_type_name
+        # Fluentd::Setting::OutS3 -> s3
+        # Override this method if not above style
+        try(:plugin_name) || self.class.to_s.split("::").last.sub(/(In|Out)/, "").downcase
+      end
+
+      def print_if_present(key)
+        # e.g.:
+        #   path /var/log/td/aaa
+        #   user nobody
+        #   retry_limit 3
+        send(key).present? ? "#{key} #{send(key)}" : ""
+      end
+
+      def boolenan(key)
+        send(key).presence == "true" ? "#{key} true" : "#{key} false"
+      end
+
+      def flag(key)
+        send(key).presence == "true" ? key.to_s : ""
+      end
+
+      def empty_value?
+        config = ""
+        self.class.const_get(:KEYS).each do |key|
+          config << conf(key)
+        end
+        config.empty?
+      end
+
+      def input_plugin?
+        self.class.to_s.match(/::In|^In/)
+      end
+
+      def output_plugin?
+        not input_plugin?
+      end
+
+      def to_config(elm_name = nil)
+        indent = "  "
+        if elm_name
+          config = "<#{elm_name}>\n"
+        else
+          if input_plugin?
+            config = "<source>\n"
+          else
+            config = "<match #{match}>\n"
+          end
+          config << "#{indent}type #{plugin_type_name}\n"
+        end
+        self.class.const_get(:KEYS).each do |key|
+          next if key == :match
+          config << indent
+          config << conf(key)
+          config << "\n"
+        end
+        if elm_name
+          config << "</#{elm_name}>\n"
+        else
+          if input_plugin?
+            config << "</source>\n"
+          else
+            config << "</match>\n"
+          end
+        end
+        config.gsub(/^[ ]*\n/m, "")
+      end
+    end
+  end
+end

data/app/models/fluentd/setting/in_syslog.rb

@@ -0,0 +1,16 @@
+class Fluentd
+  module Setting
+    class InSyslog
+      include ActiveModel::Model
+      include Common
+
+      KEYS = [
+        :port, :bind, :tag, :types
+      ].freeze
+
+      attr_accessor(*KEYS)
+
+      validates :tag, presence: true
+    end
+  end
+end

data/app/models/fluentd/setting/in_tail.rb

@@ -0,0 +1,107 @@
+class Fluentd
+  module Setting
+    class InTail
+      include ActiveModel::Model
+      attr_accessor :path, :tag, :format, :regexp, :time_format, :rotate_wait, :pos_file, :read_from_head, :refresh_interval
+
+      validates :path, presence: true
+      validates :tag, presence: true
+      #validates :format, presence: true
+
+      def self.known_formats
+        {
+          :apache2 => [:time_format],
+          :nginx => [:time_format],
+          :syslog => [:time_format],
+          :tsv => [:keys, :time_key],
+          :csv => [:keys, :time_key],
+          :ltsv => [:delimiter, :time_key],
+          :json => [:time_key],
+          :regexp => [:time_format],
+          # TODO: Grok could generate Regexp including \d, \s, etc. fluentd config parser raise error with them for escape sequence check.
+          #       TBD How to handle Grok/Regexp later, just comment out for hide
+          # :grok => [:grok_str],
+        }
+      end
+      attr_accessor *known_formats.values.flatten.compact
+
+      def known_formats
+        self.class.known_formats
+      end
+
+      def guess_format
+        case path
+        when /\.json$/
+          :json
+        when /\.csv$/
+          :csv
+        when /\.tsv$/
+          :tsv
+        when /\.ltsv$/
+          :ltsv
+        when /nginx/
+          :nginx
+        when /apache/
+          :apache2
+        when %r|/var/log|
+          :syslog
+        else
+          :regexp
+        end
+      end
+
+      def extra_format_options
+        self.class.known_formats[format.to_sym] || []
+      end
+
+      def format_specific_conf
+        return "" if %w(grok regexp).include?(format)
+
+        indent = " " * 2
+        format_specific_conf = ""
+        extra_format_options.each do |key|
+          format_specific_conf << "#{indent}#{key} #{send(key)}\n"
+        end
+        format_specific_conf
+      end
+
+      def certain_format_line
+        case format
+        when "grok"
+          "format /#{grok.convert_to_regexp(grok_str).source.gsub("/", "\\/")}/ # grok: '#{grok_str}'"
+        when "regexp"
+          "format /#{regexp}/"
+        else
+          "format #{format}"
+        end
+      end
+
+      def grok
+        @grok ||=
+          begin
+            grok = GrokConverter.new
+            grok.load_patterns
+            grok
+          end
+      end
+
+      def to_conf
+        # NOTE: Using strip_heredoc makes more complex for format_specific_conf indent
+        <<-XML.gsub(/^[ ]*\n/m, "")
+<source>
+  type tail
+  path #{path}
+  tag #{tag}
+  #{certain_format_line}
+#{format_specific_conf}
+
+  #{read_from_head.to_i.zero? ? "" : "read_from_head true"}
+  #{pos_file.present? ? "pos_file #{pos_file}" : ""}
+  #{rotate_wait.present? ? "rotate_wait #{rotate_wait}" : ""}
+  #{refresh_interval.present? ? "refresh_interval #{refresh_interval}" : ""}
+</source>
+        XML
+      end
+    end
+  end
+end