fluentd 0.12.40 → 1.6.2

data/example/copy_roundrobin.conf

@@ -0,0 +1,39 @@
+<source>
+  @type  dummy
+  @label @test
+  tag    test.copy
+  auto_increment_key id
+</source>
+
+<source>
+  @type dummy
+  @label @test
+  tag    test.rr
+  auto_increment_key id
+</source>
+
+<label @test>
+  <match test.copy>
+    @type copy
+    <store>
+      @type stdout
+      output_type json
+    </store>
+    <store>
+      @type stdout
+      output_type ltsv
+    </store>
+  </match>
+
+  <match test.rr>
+    @type roundrobin
+    <store>
+      @type stdout
+      output_type json
+    </store>
+    <store>
+      @type stdout
+      output_type ltsv
+    </store>
+  </match>
+</label>

data/example/counter.conf

@@ -0,0 +1,18 @@
+<system>
+  <counter_server>
+    scope server1
+    bind 127.0.0.1
+    port 24321
+    path tmp/back
+  </counter_server>
+</system>
+
+<source>
+  @type dummy
+  tag "test.data"
+  auto_increment_key number
+</source>
+
+<match>
+  @type stdout
+</match>

data/example/in_dummy_blocks.conf

@@ -0,0 +1,17 @@
+<source>
+  @type dummy
+  tag dummy
+  rate 100
+  dummy {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
+</source>
+
+<match dummy>
+  @type null
+  never_flush true
+  <buffer>
+    @type memory
+    overflow_action block
+    chunk_limit_size 1k
+    total_limit_size 2k
+  </buffer>
+</match>

data/example/in_dummy_with_compression.conf

@@ -0,0 +1,23 @@
+<source>
+  @type dummy
+  @label @main
+  tag "test.data"
+  size 2
+  rate 10
+  dummy {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
+  auto_increment_key number
+</source>
+
+<label @main>
+  <match test.data>
+    @type buffered_stdout
+    <buffer>
+      @type file
+      path "#{Dir.pwd}/compressed_buffers"
+      flush_at_shutdown false
+      chunk_limit_size 1m
+      flush_interval 10s
+      compress gzip
+    </buffer>
+  </match>
+</label>

data/example/in_forward.conf

@@ -1,7 +1,14 @@
+<system>
+  rpc_endpoint 0.0.0.0:24444
+</system>
+
 <source>
   @type forward
 </source>
 
 <match test>
   @type stdout
+  # <buffer>
+  #   flush_interval 10s
+  # </buffer>
 </match>

data/example/in_forward_client.conf

@@ -0,0 +1,37 @@
+<system>
+  rpc_endpoint 0.0.0.0:24444
+</system>
+
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+  <security>
+    self_hostname input.testing.local
+    shared_key    secure_communication_is_awesome
+    user_auth     yes
+    allow_anonymous_source no
+    <user>
+      username user1
+      password yes_this_is_user1
+    </user>
+    <user>
+      username user2
+      password yes_this_is_really_user2
+    </user>
+    <user>
+      username user3
+      password noooooo_this_may_not_be_user3
+    </user>
+    <client>
+      # host  127.0.0.1
+      network 127.0.0.0/24
+      shared_key using_different_key_makes_us_secure
+      users user1,user2
+    </client>
+  </security>
+</source>
+
+<match {test,test2,test3,test4}>
+  @type stdout
+</match>

data/example/in_forward_shared_key.conf

@@ -0,0 +1,15 @@
+<system>
+  rpc_endpoint 0.0.0.0:24444
+</system>
+
+<source>
+  @type forward
+  <security>
+    self_hostname input.testing.local
+    shared_key    secure_communication_is_awesome
+  </security>
+</source>
+
+<match test>
+  @type stdout
+</match>

data/example/in_forward_tls.conf

@@ -0,0 +1,14 @@
+<source>
+  @type forward
+  port 24224
+  <transport tls>
+    insecure true
+  </transport>
+</source>
+
+<match test>
+  @type stdout
+  # <buffer>
+  #   flush_interval 10s
+  # </buffer>
+</match>

data/example/in_forward_users.conf

@@ -0,0 +1,24 @@
+<system>
+  rpc_endpoint 0.0.0.0:24444
+</system>
+
+<source>
+  @type forward
+  <security>
+    self_hostname input.testing.local
+    shared_key    secure_communication_is_awesome
+    user_auth     yes
+    <user>
+      username user1
+      password yes_this_is_user1
+    </user>
+    <user>
+      username user2
+      password yes_this_is_really_user2
+    </user>
+  </security>
+</source>
+
+<match {test,test2,test3}>
+  @type stdout
+</match>

data/example/in_forward_workers.conf

@@ -0,0 +1,21 @@
+<system>
+  workers 3
+  root_dir "#{File.join(Dir.pwd, 'test', 'tmp', 'root')}"
+</system>
+
+<source>
+  @type forward
+  @id forward_in_1
+</source>
+
+<match test>
+  @type stdout
+  @id stdout_out_1
+  <inject>
+    worker_id_key worker_id
+  </inject>
+  <buffer>
+    @type file
+    flush_interval 1s
+  </buffer>
+</match>

data/example/in_http.conf

@@ -6,7 +6,9 @@
   keepalive_timeout 10
   # backlog 0
   add_http_headers false
-  format default
+  <parse>
+    @type json
+  </parse>
 </source>
 
 <match test>

data/example/in_out_forward.conf

@@ -0,0 +1,17 @@
+<source>
+  @type forward
+  port 24224
+</source>
+
+<match test.**>
+  @type forward
+  buffer_type file
+  buffer_path /tmp/fluentd.forward.buffer
+  num_threads 10
+  flush_interval 1s
+  <server>
+    host 127.0.0.1
+    port 24225
+  </server>
+</match>
+

data/example/logevents.conf

@@ -0,0 +1,25 @@
+<source>
+  @type dummy
+  @label @dummylog
+  tag "data"
+  dummy {"message":"yay"}
+</source>
+<label @dummylog>
+  <match **>
+    @type stdout
+  </match>
+</label>
+<label @FLUENT_LOG>
+  <match fluent.debug fluent.info fluent.warn fluent.error fluent.fatal>
+    @type stdout
+    <inject>
+      hostname_key "host"
+    </inject>
+  </match>
+  # <match fluent.{info,warn,error,fatal}>
+  #   @type stdout
+  #   <inject>
+  #     hostname_key "host"
+  #   </inject>
+  # </match>
+</label> 

data/example/multi_filters.conf

@@ -0,0 +1,61 @@
+# This example is to measure optimized filter pipeline performance.
+
+<source>
+  @type dummy
+  tag   test
+  size 1000
+</source>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<match test>
+  @type buffered_null
+</match>

data/example/out_exec_filter.conf

@@ -0,0 +1,42 @@
+<source>
+  @type dummy
+  @label @exec
+  tag exec_input
+  rate 10
+  auto_increment_key num
+  dummy {"data":"mydata"}
+</source>
+
+<label @exec>
+  <match exec_input>
+    @type exec_filter
+    @label @stdout
+    tag result
+    command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + "\t" + Process.pid.to_s}; 1000.times{ proc.call }'
+    num_children 3
+    child_respawn -1
+    <inject>
+      time_key time
+      time_type float
+    </inject>
+    <format>
+      @type tsv
+      keys data, num, time
+    </format>
+    <parse>
+      @type tsv
+      keys data, num, time, pid
+    </parse>
+    <extract>
+      time_key time
+      time_type float
+    </extract>
+  </match>
+</label>
+
+<label @stdout>
+  <match result>
+    @type stdout
+  </match>
+</label>
+

data/example/out_forward.conf

@@ -8,22 +8,22 @@
 
   <server>
     # first server
-    host localhost
+    host 127.0.0.1
     port 24224
   </server>
-  <server>
-    # second server
-    host localhost
-    port 24225
-  </server>
-  <server>
-    # second server
-    host localhost
-    port 24226
-    standby
-  </server>
+  # <server>
+  #   # second server
+  #   host localhost
+  #   port 24225
+  # </server>
+  # <server>
+  #   # second server
+  #   host localhost
+  #   port 24226
+  #   standby
+  # </server>
 
-  flush_interval 1
+  flush_interval 0
   send_timeout 60
   heartbeat_type udp
   heartbeat_interval 1

data/example/out_forward_buf_file.conf

@@ -0,0 +1,23 @@
+<source>
+    @type dummy
+    tag test
+</source>
+
+<source>
+    @type monitor_agent
+    emit_interval 5
+</source>
+
+<match test>
+    @type forward
+    buffer_path /tmp/fluentd.forward
+    buffer_type file
+    flush_interval 5
+    send_timeout 60
+    heartbeat_type tcp
+    heartbeat_interval 1
+    <server>
+        host 127.0.0.1
+        port 24224
+    </server>
+</match>

data/example/out_forward_client.conf

@@ -0,0 +1,109 @@
+<source>
+  @type dummy
+  tag test
+</source>
+<source>
+  @type dummy
+  tag test2
+</source>
+<source>
+  @type dummy
+  tag test3
+</source>
+<source>
+  @type dummy
+  tag test4
+</source>
+<source>
+  @type dummy
+  tag test5
+</source>
+
+<match test>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output.testing.local
+    shared_key    secure_communication_is_awesome
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user1
+    password yes_this_is_user1
+    shared_key using_different_key_makes_us_secure
+  </server>
+</match>
+
+<match test2>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output-alt1.testing.local
+    shared_key    using_different_key_makes_us_secure
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user1
+    password yes_this_is_user1
+  </server>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user2
+    password yes_this_is_really_user2
+  </server>
+</match>
+
+<match test3>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output-fail1.testing.local
+    # default key: fail
+    shared_key    secure_communication_is_awesome
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user1
+    password yes_this_is_user1
+    # [warn]: Shared key mismatch address="127.0.0.1" hostname="output-fail1.testing.local"
+  </server>
+</match>
+
+<match test4>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output-fail2.testing.local
+    shared_key    using_different_key_makes_us_secure
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user3
+    # user3 (user denied): fail
+    password noooooo_this_may_not_be_user3
+    # [warn]: Authentication failed address="127.0.0.1" hostname="output-fail2.testing.local" username="user3"
+  </server>
+</match>
+
+<match test5>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output-fail3.testing.local
+    shared_key    using_different_key_makes_us_secure
+  </security>
+  <server>
+    # another ip (host rejected): fail
+    # This pattern will work only with Ruby 2.3
+    host "#{Socket.getifaddrs.select{|i| i.addr.ipv4? }.reject{|i| i.addr.ip_address == '127.0.0.1' }.first.addr.ip_address}"
+    port 24224
+    username user1
+    password yes_this_is_user1
+    # [warn]: Anonymous client disallowed address="192.168.1.75" hostname="output-fail3.testing.local"
+  </server>
+</match>

data/example/out_forward_heartbeat_none.conf

@@ -0,0 +1,16 @@
+<source>
+  @type dummy
+  tag test
+</source>
+
+<match test>
+  @type forward
+  heartbeat_type none
+  <server>
+    host 127.0.0.1
+    port 24224
+  </server>
+  <buffer>
+    flush_mode immediate
+  </buffer>
+</match>

data/example/out_forward_shared_key.conf

@@ -0,0 +1,36 @@
+<source>
+  @type dummy
+  tag test
+</source>
+<source>
+  @type dummy
+  tag test2
+</source>
+
+<match test>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output.testing.local
+    shared_key    secure_communication_is_awesome
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+  </server>
+</match>
+
+<match test2>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output-fail.testing.local
+    shared_key    secure_communication_is_not_awesome
+    # input plugin shows warning for wrong shared_key
+    # 2016-08-08 16:27:00 +0900 [warn]: Shared key mismatch address="127.0.0.1" hostname="output-fail.testing.local"
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+  </server>
+</match>

data/example/out_forward_tls.conf

@@ -0,0 +1,18 @@
+<source>
+  @type dummy
+  tag test
+</source>
+
+<match test>
+  @type forward
+  transport tls
+  tls_insecure_mode true
+  <server>
+    # first server
+    host 127.0.0.1
+    port 24224
+  </server>
+  <buffer>
+    flush_interval 0
+  </buffer>
+</match>