fluentd 0.14.4-x86-mingw32

data/example/out_forward_buf_file.conf

@@ -0,0 +1,23 @@
+<source>
+    @type dummy
+    tag test
+</source>
+
+<source>
+    @type monitor_agent
+    emit_interval 5
+</source>
+
+<match test>
+    @type forward
+    buffer_path /tmp/fluentd.forward
+    buffer_type file
+    flush_interval 5
+    send_timeout 60
+    heartbeat_type tcp
+    heartbeat_interval 1
+    <server>
+        host 127.0.0.1
+        port 24224
+    </server>
+</match>

data/example/v0_12_filter.conf

@@ -0,0 +1,78 @@
+# An example config to use filter plugins.
+# THIS FEATURE IS SUPPORTED FOR v0.12 AND ABOVE.
+
+# in_forward to generate events to be tested.
+# You can send an arbitrary event with an arbitrary tag.
+# For example, the following command creates the event
+# {"message":"hello world"} with tag = foo
+#
+# $ echo '{"message":"hello world"}' | fluent-cat foo
+
+<source>
+  @type forward
+  port 24224
+</source>
+
+# For all events with the tag "foo", filter it out
+# UNLESS the value of the "message" field matches /keep this/
+#
+# - {"message":"keep this please"} is kept.
+# - {"message":"Do not keep"} is filtered out.
+# - {"messag22":"keep this please"} is filtered out.
+
+<filter foo>
+  @type grep
+  regexp1  message keep this
+</filter>
+
+# Matches the events that was kept by the above filter
+<match foo>
+  @type stdout
+</match>
+
+# For all events with the tag "bar", add the machine's hostname with
+# the key "hostname" BEFORE forwarding to another instance of Fluentd
+# at 123.4.2.4:24224.
+
+<filter bar>
+  @type record_transformer
+  <record>
+    hostname ${hostname}
+  </record>
+</filter>
+
+# By the time it is getting matched here, the event has
+# the "hostname" field.
+<match bar>
+  @type forward
+  <server>
+    host 123.4.2.4
+    port 24225
+  </server>
+</match>
+
+# Composing two filters. For all events with the tag foo.bar,
+# 
+# 1. The first filter filters out all events that has the field "hello"
+# 2. Then, for those events, we downcase the value of the "name" field.
+# 
+# - {"name":"SADA", "hello":100} gets filtered out
+# - {"name":"SADA"} becomes {"name":"sada"}
+# - {"last_name":"FURUHASHI"} throws an error because it has no field called "name"
+
+<filter foo.bar>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter foo.bar>
+  @type record_transformer
+  enable_ruby true
+  <record>
+    name ${name.downcase}
+  </record>
+</filter>
+
+<match foo.bar>
+  @type stdout
+</match>

data/example/v1_literal_example.conf

@@ -0,0 +1,36 @@
+<section1>
+  key1  'text'     # text
+  key2  '\''       # ' (1 char)
+  key3  '\\'       # \ (1 char)
+  key4  '\t'       # \t (2 char)
+  key5  '\['       # \[ (2 char)
+  key6  '\\['      # \[ (2 char)
+  key7  '#t'       # #t (2 char)
+  key8  '\#{test}' # \#{test} (8 char)
+  key9  '#{test}'  # #{test} (7 char)
+  key10 '\[(?<time>[^\]]*)\] (?<message>.*)'    # \[(?<time>[^\]]*\] (?<message>.*)
+</section1>
+<section2>
+  key1  "text"     # text
+  key2  "\""       # " (1 char)
+  key3  "\\"       # \ (1 char)
+  key4  "\t"       # TAB (1 char)
+  key5  "\["       # [ (1 char)
+  key6  "\\["      # \[ (2 char)
+  key7  "#t"       # #t (2 char)
+  key8  "\#{test}" # #{test} (7 char)
+  key9  "#{test}"  # replaced by eval('test')
+  key10 "\\[(?<time>[^\\]]*)\\] (?<message>.*)" # \[(?<time>[^\]]*\] (?<message>.*)
+</section2>
+<section3>
+  key1  text     # text
+  key2  \        # \ (1 char)
+  key3  \\       # \\ (2 char)
+  key4  \t       # \t (2 char)
+  key5  \[       # \[ (2 char)
+  key6  \\[      # \\[ (3 char)
+  key7  #t       # #t (2 char)
+  key8  \#{test} # \#{test} (8 char)
+  key9  #{test}  # #{test} (7 char)
+  key10 \[(?<time>[^\]]*)\] (?<message>.*)    # \[(?<time>[^\]]*\] (?<message>.*)
+</section3>

data/fluent.conf

@@ -0,0 +1,139 @@
+# In v1 configuration, type and id are @ prefix parameters.
+# @type and @id are recommended. type and id are still available for backward compatibility
+
+## built-in TCP input
+## $ echo <json> | fluent-cat <tag>
+<source>
+  @type forward
+  @id forward_input
+</source>
+
+## built-in UNIX socket input
+#<source>
+#  @type unix
+#</source>
+
+# HTTP input
+# http://localhost:8888/<tag>?json=<json>
+<source>
+  @type http
+  @id http_input
+
+  port 8888
+</source>
+
+## File input
+## read apache logs with tag=apache.access
+#<source>
+#  @type tail
+#  format apache
+#  path /var/log/httpd-access.log
+#  tag apache.access
+#</source>
+
+## Mutating event filter
+## Add hostname and tag fields to apache.access tag events
+#<filter apache.access>
+#  @type record_transformer
+#  <record>
+#    hostname ${hostname}
+#    tag ${tag}
+#  </record>
+#</filter>
+
+## Selecting event filter
+## Remove unnecessary events from apache prefixed tag events
+#<filter apache.**>
+#  @type grep
+#  include1 method GET # pass only GET in 'method' field
+#  exclude1 message debug # remove debug event
+#</filter>
+
+# Listen HTTP for monitoring
+# http://localhost:24220/api/plugins
+# http://localhost:24220/api/plugins?type=TYPE
+# http://localhost:24220/api/plugins?tag=MYTAG
+<source>
+  @type monitor_agent
+  @id monitor_agent_input
+
+  port 24220
+</source>
+
+# Listen DRb for debug
+<source>
+  @type debug_agent
+  @id debug_agent_input
+
+  bind 127.0.0.1
+  port 24230
+</source>
+
+## match tag=apache.access and write to file
+#<match apache.access>
+#  @type file
+#  path /var/log/fluent/access
+#</match>
+
+## match tag=debug.** and dump to console
+<match debug.**>
+  @type stdout
+  @id stdout_output
+</match>
+
+# match tag=system.** and forward to another fluent server
+<match system.**>
+  @type forward
+  @id forward_output
+
+  <server>
+    host 192.168.0.11
+  </server>
+  <secondary>
+    <server>
+      host 192.168.0.12
+    </server>
+  </secondary>
+</match>
+
+## match tag=myapp.** and forward and write to file
+#<match myapp.**>
+#  @type copy
+#  <store>
+#    @type forward
+#    buffer_type file
+#    buffer_path /var/log/fluent/myapp-forward
+#    retry_limit 50
+#    flush_interval 10s
+#    <server>
+#      host 192.168.0.13
+#    </server>
+#  </store>
+#  <store>
+#    @type file
+#    path /var/log/fluent/myapp
+#  </store>
+#</match>
+
+## match fluent's internal events
+#<match fluent.**>
+#  @type null
+#</match>
+
+## match not matched logs and write to file
+#<match **>
+#  @type file
+#  path /var/log/fluent/else
+#  compress gz
+#</match>
+
+## Label: For handling complex event routing
+#<label @STAGING>
+#  <match system.**>
+#    @type forward
+#    @id staging_forward_output
+#    <server>
+#      host 192.168.0.101
+#    </server>
+#  </match>
+#</label>

data/fluentd.gemspec

@@ -0,0 +1,51 @@
+require File.expand_path('../lib/fluent/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluentd"
+  gem.version       = Fluent::VERSION # see lib/fluent/version.rb
+
+  gem.authors       = ["Sadayuki Furuhashi"]
+  gem.email         = ["frsyuki@gmail.com"]
+  gem.description   = %q{Fluentd is an open source data collector designed to scale and simplify log management. It can collect, process and ship many kinds of data in near real-time.}
+  gem.summary       = %q{Fluentd event collector}
+  gem.homepage      = "http://fluentd.org/"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.has_rdoc = false
+  gem.license = "Apache-2.0"
+
+  gem.required_ruby_version = '>= 2.1'
+
+  gem.add_runtime_dependency("msgpack", [">= 0.7.0", "< 2.0.0"])
+  gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
+  gem.add_runtime_dependency("cool.io", ["~> 1.4.5"])
+  gem.add_runtime_dependency("serverengine", ["~> 2.0"])
+  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
+  gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
+  gem.add_runtime_dependency("tzinfo", ["~> 1.0"])
+  gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
+  gem.add_runtime_dependency("strptime", ["~> 0.1.7"])
+
+  # build gem for a certain platform. see also Rakefile
+  fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
+  gem.platform = fake_platform unless fake_platform.empty?
+  if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
+    gem.add_runtime_dependency("win32-service", ["~> 0.8.3"])
+    gem.add_runtime_dependency("win32-ipc", ["~> 0.6.1"])
+    gem.add_runtime_dependency("win32-event", ["~> 0.6.1"])
+    gem.add_runtime_dependency("windows-pr", ["~> 1.2.5"])
+  end
+
+  gem.add_development_dependency("rake", ["~> 11.0"])
+  gem.add_development_dependency("flexmock", ["~> 2.0"])
+  gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
+  gem.add_development_dependency("simplecov", ["~> 0.7"])
+  gem.add_development_dependency("rr", ["~> 1.0"])
+  gem.add_development_dependency("timecop", ["~> 0.3"])
+  gem.add_development_dependency("test-unit", ["~> 3.2"])
+  gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
+  gem.add_development_dependency("oj", ["~> 2.14"])
+end

data/lib/fluent/agent.rb

@@ -0,0 +1,194 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/configurable'
+require 'fluent/plugin'
+require 'fluent/output'
+
+module Fluent
+  #
+  # Agent is a resource unit who manages emittable plugins
+  #
+  # Next step: `fluentd/root_agent.rb`
+  # Next step: `fluentd/label.rb`
+  #
+  class Agent
+    include Configurable
+
+    def initialize(log:)
+      super()
+
+      @context = nil
+      @outputs = []
+      @filters = []
+
+      @lifecycle_control_list = nil
+      # lifecycle_control_list is the list of plugins in this agent, and ordered
+      # from plugins which DOES emit, then DOESN'T emit
+      # (input -> output w/ router -> filter -> output w/o router)
+      # for start: use this order DESC
+      #   (because plugins which appears later in configurations will receive events from plugins which appears ealier)
+      # for stop/before_shutdown/shutdown/after_shutdown/close/terminate: use this order ASC
+      @lifecycle_cache = nil
+
+      @log = log
+      @event_router = EventRouter.new(NoMatchMatch.new(log), self)
+      @error_collector = nil
+    end
+
+    attr_reader :log
+    attr_reader :outputs
+    attr_reader :filters
+    attr_reader :context
+    attr_reader :event_router
+    attr_reader :error_collector
+
+    def configure(conf)
+      super
+
+      # initialize <match> and <filter> elements
+      conf.elements('filter', 'match').each { |e|
+        pattern = e.arg.empty? ? '**' : e.arg
+        type = e['@type']
+        if e.name == 'filter'
+          add_filter(type, pattern, e)
+        else
+          add_match(type, pattern, e)
+        end
+      }
+    end
+
+    def lifecycle_control_list
+      return @lifecycle_control_list if @lifecycle_control_list
+
+      lifecycle_control_list = {
+        input: [],
+        output_with_router: [],
+        filter: [],
+        output: [],
+      }
+      if self.respond_to?(:inputs)
+        inputs.each do |i|
+          lifecycle_control_list[:input] << i
+        end
+      end
+      recursive_output_traverse = ->(o) {
+        if o.has_router?
+          lifecycle_control_list[:output_with_router] << o
+        else
+          lifecycle_control_list[:output] << o
+        end
+
+        if o.respond_to?(:outputs)
+          o.outputs.each do |store|
+            recursive_output_traverse.call(store)
+          end
+        end
+      }
+      outputs.each do |o|
+        recursive_output_traverse.call(o)
+      end
+      filters.each do |f|
+        lifecycle_control_list[:filter] << f
+      end
+
+      @lifecycle_control_list = lifecycle_control_list
+    end
+
+    def lifecycle(desc: false)
+      kind_list = if desc
+                    [:output, :filter, :output_with_router]
+                  else
+                    [:output_with_router, :filter, :output]
+                  end
+      kind_list.each do |kind|
+        list = if desc
+                 lifecycle_control_list[kind].reverse
+               else
+                 lifecycle_control_list[kind]
+               end
+        display_kind = (kind == :output_with_router ? :output : kind)
+        list.each do |instance|
+          yield instance, display_kind
+        end
+      end
+    end
+
+    def add_match(type, pattern, conf)
+      log.info "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
+
+      output = Plugin.new_output(type)
+      output.router = @event_router if output.respond_to?(:router=)
+      output.configure(conf)
+      @outputs << output
+      if output.respond_to?(:outputs) && (output.is_a?(Fluent::Plugin::MultiOutput) || output.is_a?(Fluent::MultiOutput))
+        @outputs.push(*output.outputs)
+      end
+      @event_router.add_rule(pattern, output)
+
+      output
+    end
+
+    def add_filter(type, pattern, conf)
+      log.info "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
+
+      filter = Plugin.new_filter(type)
+      filter.router = @event_router
+      filter.configure(conf)
+      @filters << filter
+      @event_router.add_rule(pattern, filter)
+
+      filter
+    end
+
+    # For handling invalid record
+    def emit_error_event(tag, time, record, error)
+    end
+
+    def handle_emits_error(tag, es, error)
+    end
+
+    class NoMatchMatch
+      def initialize(log)
+        @log = log
+        @count = 0
+      end
+
+      def emit_events(tag, es)
+        # TODO use time instead of num of records
+        c = (@count += 1)
+        if c < 512
+          if Math.log(c) / Math.log(2) % 1.0 == 0
+            @log.warn "no patterns matched", tag: tag
+            return
+          end
+        else
+          if c % 512 == 0
+            @log.warn "no patterns matched", tag: tag
+            return
+          end
+        end
+        @log.on_trace { @log.trace "no patterns matched", tag: tag }
+      end
+
+      def start
+      end
+
+      def shutdown
+      end
+    end
+  end
+end