fluentd-ui 0.3.14 → 0.3.15

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6bc6e3afc054e9c31d53f4cbcdf624a5747b86b8
+  data.tar.gz: 92a4781f7b6ee8a42f44e99bd7ce1b52a74c2c63
+SHA512:
+  metadata.gz: 989ffd56a8d254b321a3b7c267fff4876a0e1ae35dee49181074799866f3b4d17b61a5532f18f72acb13ef7539729a4899693ffc8a4985d8edee68087895e1cb
+  data.tar.gz: b781c709a136ab244b0d117977935e92790ed11860ec949ecaf688def91689ea6f8161724f07d4e569d3ba1f449d5a62bda41946f4c1f46b082a75e181573590

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 0.3.15 - 2015/03/11
+* [fixed] #159 Fix latest out_s3 plugin compatibility
+* [fixed] #160 Add validation that `buffer_path` required if `buffer_type` is file
+* [maintenance] Minor refactors
+
 Release 0.3.14 - 2015/02/04
 * [maintenance] #150, #151 minor fix in README.md.
 * [maintenance] #149 Make circle-ci result stable.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluentd-ui (0.3.14)
+    fluentd-ui (0.3.15)
       addressable
       bundler
       diff-lcs
@@ -141,7 +141,7 @@ GEM
     mime-types (2.4.3)
     mini_portile (0.6.2)
     minitest (5.5.1)
-    msgpack (0.5.10)
+    msgpack (0.5.11)
     multi_json (1.10.1)
     net-http-digest_auth (1.4)
     net-http-persistent (2.9.4)

data/app/controllers/concerns/setting_concern.rb

@@ -22,7 +22,7 @@ module SettingConcern
     @fluentd.agent.config_append @setting.to_config
     if @fluentd.agent.running?
       unless @fluentd.agent.restart
-        @setting.errors.add(:base, @fluentd.agent.log_tail(1).first)
+        @setting.errors.add(:base, @fluentd.agent.log.tail(1).first)
         return render "shared/settings/show"
       end
     end

data/app/controllers/concerns/setting_history_concern.rb

@@ -26,7 +26,7 @@ module SettingHistoryConcern
     if @fluentd.agent.dryrun
       flash = { success: t('messages.dryrun_is_passed') }
     else
-      flash = { danger: @fluentd.agent.log_tail(1).first }
+      flash = { danger: @fluentd.agent.log.tail(1).first }
     end
     redirect_to :back, flash: flash
   end

data/app/controllers/fluentd/agents_controller.rb

@@ -2,7 +2,7 @@ class Fluentd::AgentsController < ApplicationController
   before_action :find_fluentd
 
   def start
-    run_action(__method__) { @fluentd.agent.log_tail(1).first }
+    run_action(__method__) { @fluentd.agent.log.tail(1).first }
     redirect_to daemon_path(@fluentd), status: 303 # 303 is change HTTP Verb GET
   end
 
@@ -12,12 +12,12 @@ class Fluentd::AgentsController < ApplicationController
   end
 
   def restart
-    run_action(__method__) { @fluentd.agent.log_tail(1).first }
+    run_action(__method__) { @fluentd.agent.log.tail(1).first }
     redirect_to daemon_path(@fluentd), status: 303 # 303 is change HTTP Verb GET
   end
 
   def log_tail
-    @logs = @fluentd.agent.log_tail(params[:limit]).reverse if @fluentd
+    @logs = @fluentd.agent.log.tail(params[:limit]).reverse if @fluentd
     render json: @logs
   end
 

data/app/controllers/fluentd/settings_controller.rb

@@ -39,7 +39,7 @@ class Fluentd::SettingsController < ApplicationController
     if dryrun(params[:config])
       flash.now[:success] = I18n.t('messages.dryrun_is_passed')
     else
-      flash.now[:danger] = @fluentd.agent.last_error_message
+      flash.now[:danger] = @fluentd.agent.log.last_error_message
     end
     @config = params[:config]
     render "edit"

data/app/controllers/fluentd_controller.rb

@@ -41,11 +41,11 @@ class FluentdController < ApplicationController
 
   def errors
     @error_duration_days = 5
-    @errors = @fluentd.agent.errors_since(@error_duration_days.days.ago)
+    @errors = @fluentd.agent.log.errors_since(@error_duration_days.days.ago)
   end
 
   def raw_log
-    send_data @fluentd.agent.log, type: "application/octet-stream", filename: File.basename(@fluentd.log_file)
+    send_data @fluentd.agent.log.read, type: "application/octet-stream", filename: File.basename(@fluentd.log_file)
   end
 
   private

data/app/controllers/misc_controller.rb

@@ -38,7 +38,7 @@ class MiscController < ApplicationController
     File.unlink(path) if File.exists?(path)
 
     Zip::File.open(path, Zip::File::CREATE) do |zip|
-      zip.get_output_stream('fluentd.log') {|f| f.puts fluentd.agent.log }
+      zip.get_output_stream('fluentd.log') {|f| f.puts fluentd.agent.log.read }
       zip.add("fluentd-ui.log", log_path)
 
       add_env_file_to(zip)

data/app/controllers/tutorials_controller.rb

@@ -5,7 +5,7 @@ class TutorialsController < ApplicationController
   helper_method :tutorial_ready?
 
   def index
-    @log = @fluentd.agent.log_tail.reverse if @fluentd
+    @log = @fluentd.agent.log.tail.reverse if @fluentd
   end
 
   def chapter1

data/app/models/fluentd/agent/common.rb

@@ -19,34 +19,19 @@ class Fluentd
     module Common
       attr_reader :extra_options
 
-      def initialize(options = {})
-        @extra_options = options
-      end
-
-      def wait_process_starting_seconds
-        10.seconds # wait time for fluentd pidfile created
-      end
-
-      def errors_since(since = 1.day.ago)
-        errors = []
-        logged_errors do |error|
-          break if Time.parse(error[:subject]) < since
-          errors << error
-        end
-        errors
+      def self.included(base)
+        base.send(:include, Fluentd::Agent::ProcessOperation)
       end
 
-      def recent_errors(limit = 3)
-        errors = []
-        logged_errors do |error|
-          errors << error
-          break if errors.length >= limit
+      # define these methods on each Agent class
+      %w(start stop restart version).each do |method|
+        define_method(method) do
+          raise NotImplementedError, "'#{method}' method is required to be defined"
         end
-        errors
       end
 
-      def last_error_message
-        recent_errors(1).first.try(:[], :subject) || ""
+      def initialize(options = {})
+        @extra_options = options
       end
 
       def pid_file
@@ -57,16 +42,59 @@ class Fluentd
         extra_options[:log_file] || self.class.default_options[:log_file]
       end
 
+      def log
+        @log ||= FluentdLog.new(log_file)
+      end
+
       def config_file
         extra_options[:config_file] || self.class.default_options[:config_file]
       end
 
+      # -- config
+      def config
+        File.read(config_file)
+      end
+
+      def config_write(content)
+        backup_config
+        File.open(config_file, "w") do |f|
+          f.write content
+        end
+      end
+
+      def config_append(content)
+        backup_config
+        File.open(config_file, "a") do |f|
+          f.write "\n"
+          f.write content
+        end
+      end
+
+      def configuration
+        if File.exists? config_file
+          ::Fluentd::Agent::Configuration.new(config_file)
+        end
+      end
+
+      # -- backup methods
       def config_backup_dir
         dir = File.join(FluentdUI.data_dir, "#{Rails.env}_confg_backups")
         FileUtils.mkdir_p(dir)
         dir
       end
 
+      def backup_files
+        Dir.glob(File.join("#{config_backup_dir}", "*.conf"))
+      end
+
+      def backup_files_in_old_order
+        backup_files.sort
+      end
+
+      def backup_files_in_new_order
+        backup_files_in_old_order.reverse
+      end
+
       def running_config_backup_dir
         dir = File.join(FluentdUI.data_dir, "#{Rails.env}_running_confg_backup")
         FileUtils.mkdir_p(dir)
@@ -77,20 +105,7 @@ class Fluentd
         File.join(running_config_backup_dir, "running.conf")
       end
 
-      # define these methods on each Agent class
-
-      %w(start stop restart).each do |method|
-        define_method(method) do
-          raise NotImplementedError, "'#{method}' method is required to be defined"
-        end
-      end
-
-      %w(running? version log config config_write config_append log_tail configuration).each do |method|
-        define_method(method) do
-          raise NotImplementedError, "'#{method}' method is required to be defined"
-        end
-      end
-
+      # -------------- private --------------
       private
 
       def backup_running_config
@@ -103,6 +118,26 @@ class Fluentd
 
         true
       end
+
+      def backup_config
+        return unless File.exists? config_file
+
+        FileUtils.cp config_file, File.join(config_backup_dir, "#{Time.zone.now.strftime('%Y%m%d_%H%M%S')}.conf")
+
+        remove_over_backup_files
+      end
+
+      def remove_over_backup_files
+        over_file_count = backup_files.size - ::Settings.max_backup_files_count
+
+        return if over_file_count <= 0
+
+        backup_files_in_old_order.first(over_file_count).each do |file|
+          note_file_attached_backup = file.sub(/#{Regexp.escape(File.extname(file))}\z/, ::Fluentd::SettingArchive::Note::FILE_EXTENSION)
+          FileUtils.rm(note_file_attached_backup) if File.exist? note_file_attached_backup
+          FileUtils.rm(file) if File.exist? file
+        end
+      end
     end
   end
 end

data/app/models/fluentd/agent/fluentd_gem.rb

@@ -2,7 +2,6 @@ class Fluentd
   class Agent
     class FluentdGem
       include Common
-      include LocalCommon
 
       def self.default_options
         {
@@ -103,6 +102,10 @@ class Fluentd
           false
         end
       end
+
+      def wait_process_starting_seconds
+        10.seconds # wait time for fluentd pidfile created
+      end
     end
   end
 end

data/app/models/fluentd/agent/process_operation.rb

@@ -0,0 +1,61 @@
+class Fluentd
+  class Agent
+    module ProcessOperation
+      def self.included(base)
+        define_method(:dryrun!) do
+          raise NotImplementedError, "'dryrun!' method is required to be defined"
+        end
+      end
+
+      def running?
+        begin
+          pid && Process.kill(0, pid)
+        rescue Errno::ESRCH
+          File.unlink(pid_file) # no needed any more
+          false
+        end
+      end
+
+      def dryrun(file_path = nil)
+        dryrun!(file_path)
+        true
+      rescue ::Fluentd::Agent::ConfigError
+        false
+      end
+
+      def pid
+        return unless File.exists?(pid_file)
+        return if File.zero?(pid_file)
+        File.read(pid_file).to_i rescue nil
+      end
+
+      private
+
+      def exec_dryrun(command, file_path = nil)
+        Bundler.with_clean_env do
+          unless system("#{command} -q --dry-run #{options_to_argv(config_file: file_path)}", out: File::NULL, err: File::NULL)
+            raise ::Fluentd::Agent::ConfigError
+          end
+        end
+      end
+
+      def detached_command(cmd)
+        thread = Bundler.with_clean_env do
+          pid = spawn(cmd)
+          Process.detach(pid)
+        end
+        thread.join
+        thread.value.exitstatus.zero?
+      end
+
+      def options_to_argv(opts = {})
+        argv = ""
+        argv << " --use-v1-config"
+        argv << " -c #{opts[:config_file] || config_file}"
+        argv << " -d #{opts[:pid_file] || pid_file}"
+        argv << " -o #{opts[:log_file] || log_file}"
+        argv
+      end
+    end
+  end
+end

data/app/models/fluentd/agent/td_agent.rb

@@ -2,7 +2,6 @@ class Fluentd
   class Agent
     class TdAgent
       include Common
-      include LocalCommon
 
       def self.default_options
         {

data/app/models/fluentd/setting/out_mongo.rb

@@ -6,7 +6,7 @@ class Fluentd
       KEYS = [
         :match,
         :host, :port, :database, :collection, :capped, :capped_size, :capped_max, :user, :password, :tag_mapped,
-        :buffer_type, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval,  :retry_wait, :retry_limit, :max_retry_wait, :num_threads,
+        :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval,  :retry_wait, :retry_limit, :max_retry_wait, :num_threads,
       ].freeze
 
       attr_accessor(*KEYS)
@@ -19,6 +19,7 @@ class Fluentd
       validates :database, presence: true
       validate :validate_capped
       validate :validate_collection
+      validates :buffer_path, presence: true, if: ->{ buffer_type == "file" }
 
       def validate_capped
         return true if capped.blank?
@@ -49,7 +50,7 @@ class Fluentd
 
       def advanced_options
         [
-          :capped, :capped_size, :capped_max, :buffer_type, :buffer_queue_limit, :buffer_chunk_limit,
+          :capped, :capped_size, :capped_max, :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit,
           :flush_interval, :retry_wait, :retry_limit, :max_retry_wait, :num_threads,
         ]
       end

data/app/models/fluentd/setting/out_s3.rb

@@ -8,9 +8,9 @@ class Fluentd
         :match,
         :aws_key_id, :aws_sec_key, :s3_bucket, :s3_region, :path,
         # :reduced_redundancy, :check_apikey_on_start, :command_parameter, # not configurable?
-        :format, :include_time_key, :time_key, :delimiter, :label_delimiter, :add_newline, :output_tag, :output_time,
+        :format, :include_time_key, :time_key, :delimiter, :label_delimiter,
         :time_slice_format, :time_slice_wait, :time_format, :utc, :store_as, :proxy_uri, :use_ssl,
-        :buffer_type, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval,
+        :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval,
         :retry_wait, :retry_limit, :max_retry_wait, :num_threads,
       ].freeze
 
@@ -19,17 +19,16 @@ class Fluentd
       choice :format, %w(out_file json ltsv single_value)
       choice :store_as, %w(gzip lzo lzma2 json txt)
       choice :buffer_type, %w(memory file)
-      booleans :include_time_key, :add_newline, :use_ssl, :output_tag, :output_time
+      booleans :include_time_key, :use_ssl
       flags :utc
 
       validates :match, presence: true
       validates :s3_bucket, presence: true
+      validates :buffer_path, presence: true, if: ->{ buffer_type == "file" }
 
       def self.initial_params
         {
           s3_region: "us-west-1",
-          output_tag: true,
-          output_time: true,
           use_ssl: true,
         }
       end
@@ -43,9 +42,9 @@ class Fluentd
 
       def advanced_options
         [
-          :format, :output_tag, :output_time, :include_time_key, :time_key, :delimiter, :label_delimiter,
+          :format, :include_time_key, :time_key, :delimiter, :label_delimiter,
           :utc, :time_slice_format, :time_slice_wait, :store_as, :proxy_uri,
-          :buffer_type, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval,
+          :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval,
           :retry_wait, :retry_limit, :max_retry_wait, :num_threads,
         ]
       end

data/app/models/fluentd/setting/out_td.rb

@@ -17,6 +17,7 @@ class Fluentd
       validates :match, presence: true
       validates :apikey, presence: true
       validates :auto_create_table, presence: true
+      validates :buffer_path, presence: true, if: ->{ buffer_type == "file" }
 
       def plugin_name
         "tdlog"

data/app/models/fluentd_log.rb

@@ -0,0 +1,123 @@
+class FluentdLog
+  attr_reader :log_file
+
+  def initialize(path)
+    @log_file = path
+  end
+
+  def read
+    return "" unless File.exists?(log_file)
+    content = File.open(log_file, "r+:ascii-8bit"){|f| f.read } # TODO: large log file
+    content.force_encoding('utf-8').valid_encoding? ? content : content.force_encoding('ascii-8bit')
+  end
+
+  def errors_since(since = 1.day.ago)
+    errors = []
+    logged_errors do |error|
+      break if Time.parse(error[:subject]) < since
+      errors << error
+    end
+    errors
+  end
+
+  def recent_errors(limit = 3)
+    errors = []
+    logged_errors do |error|
+      errors << error
+      break if errors.length >= limit
+    end
+    errors
+  end
+
+  def last_error_message
+    recent_errors(1).first.try(:[], :subject) || ""
+  end
+
+  def tail(limit = nil)
+    return [] unless File.exists?(log_file)
+
+    limit = limit.to_i rescue 0
+    limit = limit.zero? ? Settings.default_log_tail_count : limit
+    io = File.open(log_file)
+    buf = []
+    reader = ::FileReverseReader.new(io)
+    reader.each_line do |line|
+      buf << line
+      break if buf.length >= limit
+    end
+    buf
+  end
+
+  private
+
+  def logged_errors(&block)
+    return [] unless File.exist?(log_file)
+    buf = []
+    io = File.open(log_file)
+    reader = ::FileReverseReader.new(io)
+    reader.each_line do |line|
+      unless line["error"]
+        if buf.present?
+          # NOTE: if a following log is given
+          #         2014-06-30 11:24:08 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=#<Errno::EADDRINUSE: Address already in use - bind(2) for 0.0.0.0:24220>
+          #         2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `bind'
+          #         2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `listen'
+          #         2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:461:in `block in tcp_server_sockets'
+          #       the first line become a "subject", trailing lines are "notes"
+          #       {
+          #         subject: "2014-06-30 11:24:08 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=#<Errno::EADDRINUSE: Address already in use - bind(2) for 0.0.0.0:24220>",
+          #         notes: [
+          #           2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `bind'
+          #           2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `listen'
+          #           2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:461:in `block in tcp_server_sockets'
+          #         ]
+          #       }
+          split_error_lines_to_error_units(buf.reverse).each do |error_unit|
+            block.call({
+              subject: error_unit[:subject],
+              notes: error_unit[:notes],
+            })
+          end
+        end
+
+        buf = []
+        next
+      end
+      buf << line
+    end
+  ensure
+    io && io.close
+  end
+
+  def split_error_lines_to_error_units(buf)
+    # NOTE: if a following log is given
+    #
+    #2014-05-27 10:54:37 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=#<Errno::#EADDRINUSE: Address already in use - bind(2) for "0.0.0.0" port 24224>
+    #2014-05-27 10:55:40 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=#<Errno::#EADDRINUSE: Address already in use - bind(2) for "0.0.0.0" port 24224>
+    #  2014-05-27 10:55:40 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.0/lib/ruby/gems/2.1.0/gems/cool.io-1.2.4/lib/cool.io/server.rb:57:in `initialize'
+    #  2014-05-27 10:55:40 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.0/lib/ruby/gems/2.1.0/gems/cool.io-1.2.4/lib/cool.io/server.rb:57:in `new'
+    #
+    #the first line and second line must be each "error_unit". and after third lines lines are "notes" of second error unit of .
+    # [
+    #   { subject: "2014-05-27 10:54:37 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=#<Errno::#EADDRINUSE: Address already in use - bind(2) for "0.0.0.0" port 24224>          ",
+    #     notes: [] },
+    #   { subject: "2014-05-27 10:55:40 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=#<Errno::#EADDRINUSE: Address already in use - bind(2) for "0.0.0.0" port 24224>          ",
+    #     notes: [
+    #       "2014-05-27 10:55:40 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.0/lib/ruby/gems/2.1.0/gems/cool.io-1.2.4/lib/cool.io/server.rb:57:in `initialize'",
+    #       "2014-05-27 10:55:40 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.0/lib/ruby/gems/2.1.0/gems/cool.io-1.2.4/lib/cool.io/server.rb:57:in `new'"
+    #     ]
+    #   },
+    # ]
+    #
+    return_array = []
+    buf.each_with_index do |b, i|
+      if b.match(/\A /)
+        return_array[-1][:notes] << b
+      else
+        return_array << { subject: b, notes: [] }
+      end
+    end
+    return return_array.reverse
+  end
+end
+