fluentd 1.2.0.pre1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e457fdb0637a4e2ca90d91766e53db7d1a35b44c
-  data.tar.gz: 71aad06bb2784901e8e1517b6433536321fdeeae
+  metadata.gz: 784878757e32d7abd7c3bed9bcb4f081524502b0
+  data.tar.gz: 89205de62e14d5b3210888a40656df87c6bc8d15
 SHA512:
-  metadata.gz: 98f12edb338eb087180087c0d10ed0d23070d06333ee39784faf24c4190ae39cce899e43f0644728a50768743b872954ad90fbd9a962caf67efe0c8ea0b12547
-  data.tar.gz: 7ce0e7b63dcbd094c3c9452851fc91c3660e44ed69fbc3cc7aa67d53cf96ac615f9c172957d0092c054250aa4c12a09837c0f1965e3caa02153a7d092070e449
+  metadata.gz: fcbe5512f4ce1ffef0af5948dd4a6a1b81095f2d3032dee86692db4eb1164f13a2710786562980d2d44f3d289f9120d53c222b4ca796eae58fbc15e0354a657a
+  data.tar.gz: 918b3cad26663a13c5ccba5e4fea2494124d793e5744e1c4e7e2b2d844a758638a3b78843e3718af7e1c95ee8420c60c06b469531dc0bba5151504817e970c64

data/CHANGELOG.md

@@ -1,5 +1,33 @@
 # v1.1
 
+## Release v1.2.0 - 2018/04/30
+
+### New Features
+
+* New Counter API
+  https://github.com/fluent/fluentd/pull/1857
+* output: Backup for broken chunks
+  https://github.com/fluent/fluentd/pull/1952
+* filter_grep: Support `<and>` and `<or>` support
+  https://github.com/fluent/fluentd/pull/1897
+* config: Support `regexp` type in configuration parameter
+  https://github.com/fluent/fluentd/pull/1927
+
+### Enhancements
+
+* parser_nginx: Support optional `http-x-forwarded-for` field
+  https://github.com/fluent/fluentd/pull/1932
+* filter_grep: Improve the performance
+  https://github.com/fluent/fluentd/pull/1940
+
+### Bug fixes
+
+* log: Fix unexpected implementation bug when log rotation setting is applied
+  https://github.com/fluent/fluentd/pull/1957
+* server helper: Close invalid socket when ssl error happens on reading
+  https://github.com/fluent/fluentd/pull/1942
+* output: Buffer chunk's unique id should be formatted as hex in the log
+
 ## Release v1.1.3 - 2018/04/03
 
 ### Enhancements

data/MAINTAINERS.md

@@ -1,6 +1,7 @@
-- Sadayuki Furuhashi <frsyuki@gmail.com>
-- Masahiro Nakagawa <repeatedly@gmail.com>
-- Kiyoto Tamura <me@ktamura.com>
-- Kazuki Ohta <kazuki.ohta@gmail.com>
-- Satoshi "Moris" Tagomori <tagomoris@gmail.com>
-- Eduardo Silva <eduardo@treasure-data.com>
+# Fluentd Maintainers
+
+- [Naotoshi Seo](https://github.com/sonots), [DeNA](http://dena.com/intl/)
+- [Okkez](https://github.com/okkez), [Clearcode](https://www.clear-code.com/)
+- [Hiroshi Hatake](https://github.com/cosmo0920), [Clearcode](https://www.clear-code.com/)
+- [Masahiro Nakagawa](https://github.com/repeatedly), [Treasure Data](https://www.treasuredata.com/)
+- [Satoshi Tagomori](https://github.com/tagomoris), [Treasure Data](https://www.treasuredata.com/)

data/example/counter.conf

@@ -0,0 +1,18 @@
+<system>
+  <counter_server>
+    scope server1
+    bind 127.0.0.1
+    port 24321
+    path tmp/back
+  </counter_server>
+</system>
+
+<source>
+  @type dummy
+  tag "test.data"
+  auto_increment_key number
+</source>
+
+<match>
+  @type stdout
+</match>

data/example/secondary_file.conf

@@ -22,9 +22,10 @@
   @type forward
 
   <buffer time,tag,message>
-    type memory
+    @type memory
     timekey 2s
     timekey_wait 1s
+    flush_mode interval
     flush_interval 1s
   </buffer>
 
@@ -34,7 +35,7 @@
   </server>
 
   <secondary>
-    type secondary_file
+    @type secondary_file
     directory log/secondary/
     basename ${tag}_%Y%m%d%L_${message}
   </secondary>

data/lib/fluent/counter.rb

@@ -0,0 +1,23 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/counter/client'
+require 'fluent/counter/server'
+
+module Fluent
+  module Counter
+  end
+end

data/lib/fluent/counter/base_socket.rb

@@ -0,0 +1,46 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'cool.io'
+require 'fluent/msgpack_factory'
+
+module Fluent
+  module Counter
+    class BaseSocket < Coolio::TCPSocket
+      include Fluent::MessagePackFactory::Mixin
+
+      def packed_write(data)
+        write pack(data)
+      end
+
+      def on_read(data)
+        msgpack_unpacker.feed_each(data) do |d|
+          on_message d
+        end
+      end
+
+      def on_message(data)
+        raise NotImplementedError
+      end
+
+      private
+
+      def pack(data)
+        msgpack_packer.pack(data)
+      end
+    end
+  end
+end

data/lib/fluent/counter/client.rb

@@ -0,0 +1,288 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'cool.io'
+require 'fluent/counter/base_socket'
+require 'timeout'
+
+module Fluent
+  module Counter
+    class Client
+      DEFAULT_PORT = 24321
+      DEFAULT_ADDR = '127.0.0.1'
+      DEFAULT_TIMEOUT = 5
+      ID_LIMIT_COUNT = 1 << 31
+
+      def initialize(loop = nil, opt = {})
+        @loop = loop || Coolio::Loop.new
+        @port = opt[:port] || DEFAULT_PORT
+        @host = opt[:host] || DEFAULT_ADDR
+        @log = opt[:log] || $log
+        @timeout = opt[:timeout] || DEFAULT_TIMEOUT
+        @conn = Connection.connect(@host, @port, method(:on_message))
+        @responses = {}
+        @id = 0
+        @id_mutex = Mutex.new
+        @loop_mutex = Mutex.new
+      end
+
+      def start
+        @loop.attach(@conn)
+        @log.debug("starting counter client: #{@host}:#{@port}")
+        self
+      rescue => e
+        if @log
+          @log.error e
+        else
+          STDERR.puts e
+        end
+      end
+
+      def stop
+        @conn.close
+        @log.debug("calling stop in counter client: #{@host}:#{@port}")
+      end
+
+      def establish(scope)
+        scope = Timeout.timeout(@timeout) {
+          response = send_request('establish', nil, [scope])
+          raise response.errors.first if response.errors?
+          data = response.data
+          data.first
+        }
+        @scope = scope
+      rescue Timeout::Error
+        raise "Can't establish the connection to counter server due to timeout"
+      end
+
+      # === Example
+      # `init` receives various arguments.
+      #
+      # 1. init(name: 'name')
+      # 2. init({ name: 'name',reset_interval: 20 }, options: {})
+      # 3. init([{ name: 'name1',reset_interval: 20 }, { name: 'name2',reset_interval: 20 }])
+      # 4. init([{ name: 'name1',reset_interval: 20 }, { name: 'name2',reset_interval: 20 }], options: {})
+      # 5. init([{ name: 'name1',reset_interval: 20 }, { name: 'name2',reset_interval: 20 }]) { |res| ... }
+      def init(params, options: {})
+        exist_scope!
+        params = [params] unless params.is_a?(Array)
+        res = send_request('init', @scope, params, options)
+
+        # if `async` is false or missing, block at this method and return a Future::Result object.
+        if block_given?
+          Thread.start do
+            yield res.get
+          end
+        else
+          res
+        end
+      end
+
+      def delete(*params, options: {})
+        exist_scope!
+        res = send_request('delete', @scope, params, options)
+
+        if block_given?
+          Thread.start do
+            yield res.get
+          end
+        else
+          res
+        end
+      end
+
+      # === Example
+      # `inc` receives various arguments.
+      #
+      # 1. inc(name: 'name')
+      # 2. inc({ name: 'name',value: 20 }, options: {})
+      # 3. inc([{ name: 'name1',value: 20 }, { name: 'name2',value: 20 }])
+      # 4. inc([{ name: 'name1',value: 20 }, { name: 'name2',value: 20 }], options: {})
+      def inc(params, options: {})
+        exist_scope!
+        params = [params] unless params.is_a?(Array)
+        res = send_request('inc', @scope, params, options)
+
+        if block_given?
+          Thread.start do
+            yield res.get
+          end
+        else
+          res
+        end
+      end
+
+      def get(*params, options: {})
+        exist_scope!
+        res = send_request('get', @scope, params, options)
+
+        if block_given?
+          Thread.start do
+            yield res.get
+          end
+        else
+          res
+        end
+      end
+
+      def reset(*params, options: {})
+        exist_scope!
+        res = send_request('reset', @scope, params, options)
+
+        if block_given?
+          Thread.start do
+            yield res.get
+          end
+        else
+          res
+        end
+      end
+
+      private
+
+      def exist_scope!
+        raise 'Call `establish` method to get a `scope` before calling this method' unless @scope
+      end
+
+      def on_message(data)
+        if response = @responses.delete(data['id'])
+          response.set(data)
+        else
+          @log.warn("Receiving missing id data: #{data}")
+        end
+      end
+
+      def send_request(method, scope, params, opt = {})
+        id = generate_id
+        res = Future.new(@loop, @loop_mutex)
+        @responses[id] = res # set a response value to this future object at `on_message`
+        request = build_request(method, id, scope, params, opt)
+        @log.debug(request)
+        @conn.send_data request
+        res
+      end
+
+      def build_request(method, id, scope = nil, params = nil, options = nil)
+        r = { id: id, method: method }
+        r[:scope] = scope if scope
+        r[:params] = params if params
+        r[:options] = options if options
+        r
+      end
+
+      def generate_id
+        id = 0
+        @id_mutex.synchronize do
+          id = @id
+          @id += 1
+          @id = 0 if ID_LIMIT_COUNT < @id
+        end
+        id
+      end
+    end
+
+    class Connection < Fluent::Counter::BaseSocket
+      def initialize(io, on_message)
+        super(io)
+        @connection = false
+        @buffer = ''
+        @on_message = on_message
+      end
+
+      def send_data(data)
+        if @connection
+          packed_write data
+        else
+          @buffer += pack(data)
+        end
+      end
+
+      def on_connect
+        @connection = true
+        write @buffer
+        @buffer = ''
+      end
+
+      def on_close
+        @connection = false
+      end
+
+      def on_message(data)
+        @on_message.call(data)
+      end
+    end
+
+    class Future
+      class Result
+        attr_reader :data, :errors
+
+        def initialize(result)
+          @errors = result['errors']
+          @data = result['data']
+        end
+
+        def success?
+          @errors.nil? || @errors.empty?
+        end
+
+        def error?
+          !success?
+        end
+      end
+
+      def initialize(loop, mutex)
+        @set = false
+        @result = nil
+        @mutex = mutex
+        @loop = loop
+      end
+
+      def set(v)
+        @result = Result.new(v)
+        @set = true
+      end
+
+      def errors
+        get.errors
+      end
+
+      def errors?
+        es = errors
+        es && !es.empty?
+      end
+
+      def data
+        get.data
+      end
+
+      def get
+        # Block until `set` method is called and @result is set
+        join if @result.nil?
+        @result
+      end
+
+      private
+
+      def join
+        until @set
+          @mutex.synchronize do
+            @loop.run_once(0.0001) # retun a lock as soon as possible
+          end
+        end
+      end
+    end
+  end
+end