fluentd 0.12.43 → 0.14.0

data/README.md

@@ -26,31 +26,6 @@ Mobile/Web Application Logging | Fluentd can function as middleware to enable as
     $ fluentd -c conf/fluent.conf &
     $ echo '{"json":"message"}' | fluent-cat debug.test
 
-## Development
-
-### Prerequisites
-
-- Ruby 2.1 or later
-- git
-
-`git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.
-
-### Install dependent gems
-
-Use bundler:
-
-    $ gem install bundler
-    $ bundle install --path vendor/bundle
-
-### Run test
-
-    $ bundle exec rake test
-
-You can run specified test via `TEST` environment variable:
-
-    $ bundle exec rake test TEST=test/test_specified_path.rb
-    $ bundle exec rake test TEST=test/test_*.rb
-
 ## Fluentd UI: Admin GUI
 
 [Fluentd UI](https://github.com/fluent/fluentd-ui) is a graphical user interface to start/stop/configure Fluentd.

data/Rakefile

@@ -15,7 +15,8 @@ Rake::TestTask.new(:base_test) do |t|
   t.libs << "test"
   t.test_files = Dir["test/**/test_*.rb"].sort
   t.verbose = true
-  #t.warning = true
+  t.warning = true
+  t.ruby_opts = ["-Eascii-8bit:ascii-8bit"]
 end
 
 task :parallel_test do

data/Vagrantfile

@@ -0,0 +1,17 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure(2) do |config|
+  config.vm.box = "ubuntu/trusty64"
+  config.vm.network "private_network", type: "dhcp"
+  config.vm.provision "shell", inline: <<-SHELL
+    sudo apt-get install -y software-properties-common
+    sudo apt-add-repository ppa:brightbox/ruby-ng
+    sudo apt-get update
+    sudo apt-get install -y build-essential git ruby2.3 ruby2.3-dev
+    sudo apt-get install -y ruby-switch
+    sudo ruby-switch --set ruby2.3
+    gem install bundler
+    (cd /vagrant; bundle install)
+  SHELL
+end

data/appveyor.yml

@@ -0,0 +1,35 @@
+version: '{build}'
+
+# init:
+#   - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+
+install:
+  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
+  - "%devkit%\\devkitvars.bat"
+  - ruby --version
+  - gem --version
+  - bundle install
+build: off
+test_script:
+  - bundle exec rake test TESTOPTS=-v
+
+branches:
+  only:
+    - master
+    - v0.10
+    - v0.12
+    - v0.14
+
+environment:
+  matrix:
+    - ruby_version: "22-x64"
+      devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "22"
+      devkit: C:\Ruby23\DevKit
+    - ruby_version: "21-x64"
+      devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "21"
+      devkit: C:\Ruby23\DevKit
+matrix:
+  allow_failures:
+    - ruby_version: "21"

data/example/filter_stdout.conf

@@ -1,22 +1,22 @@
 <source>
-  @type dummy
+  type dummy
   tag dummy
 </source>
 
 <filter **>
-  @type stdout
+  type stdout
 </filter>
 
 <filter **>
-  @type stdout
+  type stdout
   output_type hash
 </filter>
 
 <filter **>
-  @type stdout
+  type stdout
   format ltsv
 </filter>
 
 <match **>
-  @type null
+  type null
 </match>

data/example/in_forward.conf

@@ -1,7 +1,7 @@
 <source>
-  @type forward
+  type forward
 </source>
 
 <match test>
-  @type stdout
+  type stdout
 </match>

data/example/in_http.conf

@@ -1,5 +1,5 @@
 <source>
-  @type http
+  type http
   bind 0.0.0.0
   port 9880
   body_size_limit 32MB
@@ -10,5 +10,5 @@
 </source>
 
 <match test>
-  @type stdout
+  type stdout
 </match>

data/example/in_out_forward.conf

@@ -0,0 +1,17 @@
+<source>
+  @type forward
+  port 24224
+</source>
+
+<match test.**>
+  @type forward
+  buffer_type file
+  buffer_path /tmp/fluentd.forward.buffer
+  num_threads 10
+  flush_interval 1s
+  <server>
+    host 127.0.0.1
+    port 24225
+  </server>
+</match>
+

data/example/in_syslog.conf

@@ -1,5 +1,5 @@
 <source>
-  @type syslog
+  type syslog
   bind 0.0.0.0
   port 5140
   tag test
@@ -11,5 +11,5 @@
 </source>
 
 <match test>
-  @type stdout
+  type stdout
 </match>

data/example/in_tail.conf

@@ -1,5 +1,5 @@
 <source>
-  @type tail
+  type tail
   format none
   path /var/log/fluentd_test.log
   pos_file /var/log/fluentd_test.pos
@@ -10,5 +10,5 @@
 </source>
 
 <match test>
-  @type stdout
+  type stdout
 </match>

data/example/in_tcp.conf

@@ -1,5 +1,5 @@
 <source>
-  @type tcp
+  type tcp
   format none
   bind 0.0.0.0
   port 5170
@@ -9,5 +9,5 @@
 </source>
 
 <match test>
-  @type stdout
+  type stdout
 </match>

data/example/in_udp.conf

@@ -1,5 +1,5 @@
 <source>
-  @type udp
+  type udp
   format none
   bind 0.0.0.0
   port 5160
@@ -9,5 +9,5 @@
 </source>
 
 <match test>
-  @type stdout
+  type stdout
 </match>

data/example/out_copy.conf

@@ -1,15 +1,15 @@
 <source>
-  @type forward
+  type forward
 </source>
 
 <match test>
-  @type copy
+  type copy
   deep_copy false
   <store>
-    @type stdout
+    type stdout
   </store>
   <store>
-    @type file
+    type file
     path /var/log/fluentd/out_file_test
     format json
     buffer_type memory

data/example/out_file.conf

@@ -1,9 +1,9 @@
 <source>
-  @type forward
+  type forward
 </source>
 
 <match test>
-  @type file
+  type file
   path /var/log/fluentd/out_file_test
   format json
   buffer_type memory

data/example/out_forward.conf

@@ -1,10 +1,10 @@
 <source>
-  @type dummy
+  type dummy
   tag test
 </source>
 
 <match test>
-  @type forward
+  type forward
 
   <server>
     # first server

data/example/out_forward_buf_file.conf

@@ -0,0 +1,23 @@
+<source>
+    @type dummy
+    tag test
+</source>
+
+<source>
+    @type monitor_agent
+    emit_interval 5
+</source>
+
+<match test>
+    @type forward
+    buffer_path /tmp/fluentd.forward
+    buffer_type file
+    flush_interval 5
+    send_timeout 60
+    heartbeat_type tcp
+    heartbeat_interval 1
+    <server>
+        host 127.0.0.1
+        port 24224
+    </server>
+</match>

data/example/v0_12_filter.conf

@@ -9,7 +9,7 @@
 # $ echo '{"message":"hello world"}' | fluent-cat foo
 
 <source>
-  @type forward
+  type forward
   port 24224
 </source>
 
@@ -21,13 +21,13 @@
 # - {"messag22":"keep this please"} is filtered out.
 
 <filter foo>
-  @type grep
+  type grep
   regexp1  message keep this
 </filter>
 
 # Matches the events that was kept by the above filter
 <match foo>
-  @type stdout
+  type stdout
 </match>
 
 # For all events with the tag "bar", add the machine's hostname with
@@ -35,7 +35,7 @@
 # at 123.4.2.4:24224.
 
 <filter bar>
-  @type record_transformer
+  type record_transformer
   <record>
     hostname ${hostname}
   </record>
@@ -44,7 +44,7 @@
 # By the time it is getting matched here, the event has
 # the "hostname" field.
 <match bar>
-  @type forward
+  type forward
   <server>
     host 123.4.2.4
     port 24225
@@ -61,12 +61,12 @@
 # - {"last_name":"FURUHASHI"} throws an error because it has no field called "name"
 
 <filter foo.bar>
-  @type grep
+  type grep
   exclude1 hello .
 </filter>
 
 <filter foo.bar>
-  @type record_transformer
+  type record_transformer
   enable_ruby true
   <record>
     name ${name.downcase}
@@ -74,5 +74,5 @@
 </filter>
 
 <match foo.bar>
-  @type stdout
+  type stdout
 </match>

data/fluent.conf

@@ -31,6 +31,24 @@
 #  tag apache.access
 #</source>
 
+## Mutating event filter
+## Add hostname and tag fields to apache.access tag events
+#<filter apache.access>
+#  @type record_transformer
+#  <record>
+#    hostname ${hostname}
+#    tag ${tag}
+#  </record>
+#</filter>
+
+## Selecting event filter
+## Remove unnecessary events from apache prefixed tag events
+#<filter apache.**>
+#  @type grep
+#  include1 method GET # pass only GET in 'method' field
+#  exclude1 message debug # remove debug event
+#</filter>
+
 # Listen HTTP for monitoring
 # http://localhost:24220/api/plugins
 # http://localhost:24220/api/plugins?type=TYPE
@@ -108,3 +126,14 @@
 #  path /var/log/fluent/else
 #  compress gz
 #</match>
+
+## Label: For handling complex event routing
+#<label @STAGING>
+#  <match system.**>
+#    @type forward
+#    @id staging_forward_output
+#    <server>
+#      host 192.168.0.101
+#    </server>
+#  </match>
+#</label>

data/fluentd.gemspec

@@ -17,25 +17,32 @@ Gem::Specification.new do |gem|
   gem.has_rdoc = false
   gem.license = "Apache-2.0"
 
-  gem.required_ruby_version = '>= 1.9.3'
+  gem.required_ruby_version = '>= 2.1'
 
-  gem.add_runtime_dependency("msgpack", [">= 0.5.11", "< 2"])
+  gem.add_runtime_dependency("msgpack", [">= 0.7.0"])
   gem.add_runtime_dependency("json", [">= 1.4.3"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
-  gem.add_runtime_dependency("cool.io", [">= 1.2.2", "< 2.0.0"])
+  gem.add_runtime_dependency("cool.io", [">= 1.4.3", "< 2.0.0"])
+  gem.add_runtime_dependency("serverengine", [">= 1.6.4"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0.0"])
   gem.add_runtime_dependency("tzinfo-data", [">= 1.0.0"])
-  gem.add_runtime_dependency("string-scrub", [">= 0.0.3", "<= 0.0.5"])
+  if /mswin|mingw/ =~ RUBY_PLATFORM
+    gem.add_runtime_dependency("win32-service", ["~> 0.8.3"])
+    gem.add_runtime_dependency("win32-ipc", ["~> 0.6.1"])
+    gem.add_runtime_dependency("win32-event", ["~> 0.6.1"])
+    gem.add_runtime_dependency("windows-pr", ["~> 1.2.3"])
+  end
+  gem.add_runtime_dependency("strptime", [">= 0.1.7"])
 
   gem.add_development_dependency("rake", [">= 0.9.2"])
-  gem.add_development_dependency("flexmock", ["~> 2.0"])
-  gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
-  gem.add_development_dependency("simplecov", ["~> 0.7"])
-  gem.add_development_dependency("rr", ["~> 1.0"])
-  gem.add_development_dependency("timecop", ["~> 0.3"])
-  gem.add_development_dependency("test-unit", ["~> 3.2"])
-  gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
+  gem.add_development_dependency("flexmock", ["~> 2.0.5"])
+  gem.add_development_dependency("parallel_tests", [">= 0.15.3"])
+  gem.add_development_dependency("simplecov", ["~> 0.6.4"])
+  gem.add_development_dependency("rr", [">= 1.0.0"])
+  gem.add_development_dependency("timecop", [">= 0.3.0"])
+  gem.add_development_dependency("test-unit", ["~> 3.1.4"])
+  gem.add_development_dependency("test-unit-rr", ["~> 1.0.3"])
   gem.add_development_dependency("oj", ["~> 2.14"])
 end