fluentd-ui 1.0.0.alpha.2 → 1.0.0.alpha.3

data/app/views/shared/vue/_parser_multiline_form.html.haml

@@ -0,0 +1,19 @@
+%script{type: "text/x-template", id: "vue-parser-multiline-form"}
+  .form-group
+    %label{"for" => "setting_parse_0__format_firstline",
+           "data-toggle" => "tooltip",
+           "data-placement" => "right",
+           "v-bind:title" => "formatFirstlineDesc"}
+      Format first line
+    %input.form-control{"type" => "text",
+                        "id" => "setting_parse_0__format_firstline",
+                        "name" => "setting[parse[0]][format_firstline]",
+                        "v-model.lazy" => "formatFirstline"}
+    %label.mt-3{"v-bind:for" => "textareaId(pluginType)"}
+      Formats
+    %p.alert.alert-warning
+      = t("fluentd.settings.in_tail.notice_for_multiline_limit")
+    %textarea.form-control{"v-bind:id" => "textareaId(pluginType)",
+                           "v-bind:name" => "textareaName(pluginType)",
+                           "v-model.lazy" => "formats",
+                           "rows" => 20}

data/app/views/shared/vue/_setting.html.erb

@@ -4,9 +4,9 @@
 
 <div class="card card-primary mb-3">
   <div class="card-header" data-toggle="collapse" v-bind:href="'#'+id" v-bind:title="content">
+    <i class="fa fa-caret-down"></i>
     {{ type }}
     <span v-if="name == 'match'">({{ arg }})</span>
-    <i class="fa fa-caret-down"></i>
   </div>
   <div class="card-body collapse" v-bind:id="id">
     <pre v-if="mode != 'edit'">{{ content }}</pre>

data/config/application.rb

@@ -46,7 +46,7 @@ module FluentdUi
     # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
     config.i18n.default_locale = 'en'
     config.i18n.available_locales = %i(en ja)
-    config.autoload_paths += %W(#{config.root}/lib)
+    config.autoload_paths += %W(#{config.root}/lib #{Rails.root}/app/form_builders)
 
     config.active_job.queue_adapter = :sucker_punch
 

data/config/environment.rb

@@ -1,5 +1,8 @@
 # Load the Rails application.
 require_relative 'application'
 
+# Load fluentd libraries & plugins
+require "fluent/load"
+
 # Initialize the Rails application.
 Rails.application.initialize!

data/config/initializers/types.rb

@@ -0,0 +1,7 @@
+ActiveModel::Type.register(:array, Fluentd::Setting::Type::Array)
+ActiveModel::Type.register(:enum, Fluentd::Setting::Type::Enum)
+ActiveModel::Type.register(:bool, Fluentd::Setting::Type::Bool)
+ActiveModel::Type.register(:hash, Fluentd::Setting::Type::Hash)
+ActiveModel::Type.register(:regexp, Fluentd::Setting::Type::Hash)
+ActiveModel::Type.register(:size, Fluentd::Setting::Type::Size)
+ActiveModel::Type.register(:section, Fluentd::Setting::Type::Section)

data/config/routes.rb

@@ -49,7 +49,7 @@ Rails.application.routes.draw do
           post "finish"
         end
 
-        resource :out_td, only: [:show], module: :settings, controller: :out_td do
+        resource :out_tdlog, only: [:show], module: :settings, controller: :out_tdlog do
           post "finish"
         end
 
@@ -124,5 +124,6 @@ Rails.application.routes.draw do
     post "grok_to_regexp"
 
     resources :settings, only: [:index, :show, :update, :destroy], defaults: { format: "json" }
+    resources :config_definitions, only: [:index], defaults: { format: "json" }
   end
 end

data/fluentd-ui.gemspec

@@ -53,4 +53,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rubyzip", "~> 1.1" # API changed as Zip::ZipFile -> Zip::File since v1.0.0
   spec.add_dependency "diff-lcs"
   spec.add_dependency "webpacker"
+
+  spec.add_dependency "fluent-plugin-td", "~> 1.0"
+  spec.add_dependency "fluent-plugin-mongo", "~> 1.1"
+  spec.add_dependency "fluent-plugin-elasticsearch", "~> 2.10"
+  spec.add_dependency "fluent-plugin-s3", "~> 1.1"
 end

data/lib/fluentd-ui/version.rb

@@ -1,3 +1,3 @@
 module FluentdUI
-  VERSION = "1.0.0.alpha.2"
+  VERSION = "1.0.0.alpha.3"
 end

data/lib/regexp_preview.rb

@@ -6,8 +6,8 @@ require "regexp_preview/single_line"
 require "regexp_preview/multi_line"
 
 module RegexpPreview
-  def self.processor(format)
-    case format
+  def self.processor(parse_type)
+    case parse_type
     when "multiline"
       RegexpPreview::MultiLine
     else

data/lib/regexp_preview/multi_line.rb

@@ -1,80 +1,70 @@
 module RegexpPreview
   class MultiLine
-    attr_reader :file, :format, :params
+    attr_reader :path, :plugin_name, :plugin_config, :plugin
 
-    def initialize(file, format, params = {})
-      @file = file
-      @format = format
-      @params = params[:params]
+    def initialize(path, plugin_name, plugin_config = {})
+      @path = path
+      @plugin_name = plugin_name
+      @plugin_config = plugin_config
+
+      config = Fluent::Config::Element.new("ROOT", "", @plugin_config, [])
+      @plugin = Fluent::Plugin.new_parser(@plugin_name).tap do |instance|
+        instance.configure(config)
+      end
     end
 
-    def matches_json
+    def matches
       {
-        params: {
-          setting: { # for vue.js
-            regexp: nil,
-            time_format: nil,
-          }
-        },
-        matches: matches.compact,
+        pluginConfig: @plugin_config,
+        matches: _matches,
       }
     end
 
     private
 
-    def matches
-      return [] if patterns.empty?
-      reader = FileReverseReader.new(File.open(file))
-      result = []
-      target_lines = reader.tail(Settings.in_tail_preview_line_count).map{|line| line << "\n" }
-      whole_string = target_lines.join
-      re_firstline = Regexp.new(params[:format_firstline])
-      indexes = []
-      cur = 0
-      while first_index = whole_string.index(re_firstline, cur)
-        indexes << first_index
-        cur = first_index + 1
-      end
-      indexes.each_with_index do |index, i|
-        next_index = indexes[i + 1] || -1
-        chunk = whole_string[index...next_index]
-        ret = detect_chunk(chunk)
-        next unless ret
-        result << ret
-      end
-      result
-    end
-
-    def detect_chunk(chunk)
-      whole = ""
-      matches = []
-      offset = 0
-      patterns.each do |pat|
-        match = chunk.match(pat)
-        return nil unless match
-        offset = chunk.index(pat)
-        return nil if offset > 0
-        chunk = chunk[match[0].length..-1]
-        match.names.each_with_index do |name, index|
-          matches << {
-            key: name,
-            matched: match[name],
-            pos: match.offset(index + 1).map{|pos| pos + whole.length},
+    def _matches
+      begin
+        io = File.open(path)
+        reader = FileReverseReader.new(io)
+        parserd_chunks = []
+        target_lines = reader.tail(Settings.in_tail_preview_line_count).map{|line| line << "\n" }
+        whole_string = target_lines.join
+        firstline_regex = Regexp.new(plugin_config["format_firstline"][1..-2])
+        indexes = []
+        cur = 0
+        while first_index = whole_string.index(firstline_regex, cur)
+          indexes << first_index
+          cur = first_index + 1
+        end
+        indexes.each_with_index do |index, i|
+          next_index = indexes[i + 1] || -1
+          chunk = whole_string[index...next_index]
+          parsed = {
+            whole: chunk,
+            matches: []
           }
+          @plugin.parse(chunk) do |time, record|
+            next unless record
+            last_pos = 0
+            record.each do |key, value|
+              start = chunk.index(value, last_pos)
+              finish = start + value.bytesize
+              last_pos = finish
+              parsed[:matches] << {
+                key: key,
+                matched: value,
+                pos: [start, finish]
+              }
+            end
+          end
+          parserd_chunks << parsed
         end
-        whole << match[0]
+        parserd_chunks.reject do |parsed|
+          parsed[:matches].blank?
+        end
+      ensure
+        io.close
       end
-      {
-        whole: whole,
-        matches: matches,
-      }
-    end
-
-    def patterns
-      @patterns ||= (1..20).map do |n|
-        params["format#{n}"].presence
-      end.compact.map {|pattern| Regexp.new(pattern, Regexp::MULTILINE)}
     end
   end
 end
-

data/lib/regexp_preview/single_line.rb

@@ -1,65 +1,59 @@
 module RegexpPreview
   class SingleLine
-    attr_reader :file, :format, :params, :regexp, :time_format
+    attr_reader :path, :plugin_name, :plugin_config, :plugin
 
-    def initialize(file, format, params = {})
-      @file = file
-      @format = format
-      @time_format = params[:time_format]
-      @params = params
+    def initialize(path, plugin_name, plugin_config = {})
+      @path = path
+      @plugin_name = plugin_name.to_sym
+      @plugin_config = plugin_config
 
-      case format
-      when "regexp"
-        @regexp = Regexp.new(params[:regexp])
-        @time_format = nil
-      when "ltsv", "json", "csv", "tsv"
-        @regexp = nil
-        @time_format = nil
-      else # apache, nginx, etc
-        parser_plugin = Fluent::Plugin.new_parser(format)
-        raise "Unknown format '#{format}'" unless parser_plugin
-        parser_plugin.configure(Fluent::Config::Element.new('ROOT', '', {}, [])) # NOTE: SyslogParser define @regexp in configure method so call it to grab Regexp object
-        @regexp = parser_plugin.instance_variable_get(:@regexp)
-        @time_format = parser_plugin.time_format
+      config = Fluent::Config::Element.new("ROOT", "", @plugin_config, [])
+      @plugin = Fluent::Plugin.new_parser(@plugin_name).tap do |instance|
+        instance.configure(config)
       end
     end
 
-    def matches_json
+    def matches
       {
-        params: {
-          setting: {
-            # NOTE: regexp and time_format are used when format == 'apache' || 'nginx' || etc.
-            regexp: regexp.try(:source),
-            time_format: time_format,
-          }
-        },
-        matches: matches.compact,
+        pluginConfig: @plugin_config,
+        matches: _matches
       }
     end
 
     private
 
-    def matches
-      return [] unless @regexp # such as ltsv, json, etc
-      reader = FileReverseReader.new(File.open(file))
-      matches = reader.tail(Settings.in_tail_preview_line_count).map do |line|
-        result = {
-          :whole => line,
-          :matches => [],
-        }
-        match = line.match(regexp)
-        next result unless match
-
-        match.names.each_with_index do |name, index|
-          result[:matches] << {
-            key: name,
-            matched: match[name],
-            pos: match.offset(index + 1),
+    def _matches
+      return [] if %i(json csv tsv ltsv).include?(@plugin_name)
+      begin
+        io = File.open(path)
+        reader = FileReverseReader.new(io)
+        parsed_lines = reader.tail(Settings.in_tail_preview_line_count).map do |line|
+          parsed = {
+            whole: line,
+            matches: []
           }
+          @plugin.parse(line) do |time, record|
+            next unless record
+            last_pos = 0
+            record.each do |key, value|
+              start = line.index(value, last_pos)
+              finish = start + value.bytesize
+              last_pos = finish
+              parsed[:matches] << {
+                key: key,
+                matched: value,
+                pos: [start, finish]
+              }
+            end
+          end
+          parsed
+        end
+        parsed_lines.reject do |parsed|
+          parsed[:matches].blank?
         end
-        result
+      ensure
+        io.close
       end
-      matches
     end
   end
 end

data/spec/features/fluentd/setting/out_forward_spec.rb

@@ -6,7 +6,7 @@ describe "out_forward", stub: :daemon do
   let(:type) { "out_forward" }
   let(:page_url) { send("daemon_setting_#{type}_path") }
   let(:form_values) { {
-    Match: "*",
+    Pattern: "*",
     Name: "name",
     Host: "host",
     Port: "9999",

data/spec/features/fluentd/setting/out_stdout_spec.rb

@@ -2,6 +2,6 @@ require "spec_helper"
 
 describe "out_stdout", stub: :daemon do
   before { login_with exists_user }
-  it_should_behave_like "configurable daemon settings", "out_stdout", "match", "stdout.**"
+  it_should_behave_like "configurable daemon settings", "out_stdout", "pattern", "stdout.**"
 
 end

data/spec/features/{out_td_spec.rb → out_tdlog_spec.rb}

@@ -1,6 +1,6 @@
 require "spec_helper"
 
-describe "out_td", stub: :daemon do
+describe "out_tdlog", stub: :daemon do
   let(:exists_user) { build(:user) }
   let(:api_key) { "dummydummy" }
 
@@ -9,13 +9,13 @@ describe "out_td", stub: :daemon do
   end
 
   it "Shown form with filled in td.*.* on match" do
-    visit daemon_setting_out_td_path
+    visit daemon_setting_out_tdlog_path
     page.should have_css('input[name="fluentd_setting_out_td[match]"]')
   end
 
   it "Updated config after submit" do
     daemon.agent.config.should_not include(api_key)
-    visit daemon_setting_out_td_path
+    visit daemon_setting_out_tdlog_path
     within('#new_fluentd_setting_out_td') do
       fill_in "Apikey", with: api_key
     end

data/spec/lib/regexp_preview/multi_line_spec.rb

@@ -1,33 +1,29 @@
 require 'spec_helper'
 
 describe RegexpPreview::MultiLine do
-  describe "#matches_json" do
-    subject { parser.matches_json }
-    let(:parser) { RegexpPreview::MultiLine.new(target_path, "multiline", params) }
+  describe "#matches" do
+    subject { parser.matches }
+    let(:parser) { RegexpPreview::MultiLine.new(target_path, "multiline", plugin_config) }
 
     describe "simple usage" do
       let(:target_path) { File.expand_path("./spec/support/fixtures/error0.log", Rails.root) }
 
-      let :params do
-        params = {
-          format_firstline: "foo",
-          time_format: "time_format",
+      let :plugin_config do
+        plugin_config = {
+          "format_firstline" => "/foo/",
+          "time_format" => "time_format",
         }
-        params["format1"] = "(?<foo>foo)\n"
-        params["format2"] = "(?<bar>bar)"
+        plugin_config["format1"] = "/(?<foo>foo)\n/"
+        plugin_config["format2"] = "/(?<bar>bar)/"
         3.upto(Fluentd::Setting::InTail::MULTI_LINE_MAX_FORMAT_COUNT) do |i|
-          params["format#{i}"] = ""
+          plugin_config["format#{i}"] = "//"
         end
-        { params: params }
-      end
-
-      it 'should not have regexp and time_format in [:params][:setting]' do
-        expect(subject[:params][:setting]).to eq(regexp: nil, time_format: nil)
+        plugin_config
       end
 
       it "should include matches info" do
         matches_info = {
-          whole: "foo\nbar",
+          whole: "foo\nbar\nbaz\n1\n2\n3\n4\n5\n6\n10\n11\n12",
           matches: [
             {
               key: "foo", matched: "foo", pos: [0, 3]
@@ -37,24 +33,23 @@ describe RegexpPreview::MultiLine do
             }
           ]
         }
-
         expect(subject[:matches]).to include matches_info
       end
     end
 
     describe "detect only continuos patterns" do
       let(:target_path) { File.expand_path("./spec/support/fixtures/error0.log", Rails.root) }
-      let(:params) do
-        params = {
-          format_firstline: "foo",
-          time_format: "time_format",
+      let(:plugin_config) do
+        plugin_config = {
+          "format_firstline" => "/foo/",
+          "time_format" => "time_format",
         }
-        params["format1"] = "(?<foo>foo)\n"
-        params["format2"] = "(?<bar>baz)"
+        plugin_config["format1"] = "/(?<foo>foo)\n/"
+        plugin_config["format2"] = "/(?<bar>baz)/"
         3.upto(Fluentd::Setting::InTail::MULTI_LINE_MAX_FORMAT_COUNT) do |i|
-          params["format#{i}"] = ""
+          plugin_config["format#{i}"] = "//"
         end
-        { params: params }
+        plugin_config
       end
 
       it "shouldn't match" do
@@ -66,21 +61,21 @@ describe RegexpPreview::MultiLine do
       # http://docs.fluentd.org/articles/in_tail
       let(:target_path) { File.expand_path("./spec/support/fixtures/multiline_example.log", Rails.root) }
 
-      let :params do
-        params = {
-          format_firstline: "\\d{4}-\\d{1,2}-\\d{1,2}",
-          "format1" => "^(?<time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{1,2}:\\d{1,2}:\\d{1,2}) \\[(?<thread>.*)\\] (?<level>[^\\s]+)(?<message>.*)",
-          time_format: "time_format",
+      let :plugin_config do
+        plugin_config = {
+          "format_firstline" => "/\\d{4}-\\d{1,2}-\\d{1,2}/",
+          "format1" => "/^(?<time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{1,2}:\\d{1,2}:\\d{1,2}) \\[(?<thread>.*)\\] (?<level>[^\\s]+)(?<message>.*)/",
+          "time_format" => "%Y-%m-%d %H:%M:%S",
+          "keep_time_key" => true
         }
         2.upto(Fluentd::Setting::InTail::MULTI_LINE_MAX_FORMAT_COUNT) do |i|
-          params["format#{i}"] = ""
+          plugin_config["format#{i}"] = "//"
         end
-        { params: params }
+        plugin_config
       end
 
       it "should include matches info" do
-        matches_info = 
-          [
+        matches_info = [
             {
               whole: "2013-3-03 14:27:33 [main] INFO  Main - Start\n",
               matches: [