fluentd 0.12.40 → 1.6.2

data/lib/fluent/counter/server.rb

@@ -0,0 +1,273 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'cool.io'
+require 'fluent/counter/base_socket'
+require 'fluent/counter/validator'
+require 'fluent/counter/store'
+require 'fluent/counter/mutex_hash'
+
+module Fluent
+  module Counter
+    class Server
+      DEFAULT_ADDR = '127.0.0.1'
+      DEFAULT_PORT = 24321
+
+      def initialize(name, opt = {})
+        raise 'Counter server name is invalid' unless Validator::VALID_NAME =~ name
+        @name = name
+        @opt = opt
+        @addr = @opt[:addr] || DEFAULT_ADDR
+        @port = @opt[:port] || DEFAULT_PORT
+        @loop = @opt[:loop] || Coolio::Loop.new
+        @log =  @opt[:log] || $log
+
+        @store = Fluent::Counter::Store.new(opt)
+        @mutex_hash = MutexHash.new(@store)
+
+        @server = Coolio::TCPServer.new(@addr, @port, Handler, method(:on_message))
+        @thread = nil
+        @running = false
+      end
+
+      def start
+        @server.attach(@loop)
+        @thread = Thread.new do
+          @running = true
+          @loop.run(0.5)
+          @running = false
+        end
+        @log.debug("starting counter server #{@addr}:#{@port}")
+        @mutex_hash.start
+        self
+      end
+
+      def stop
+        # This `sleep` for a test to wait for a `@loop` to begin to run
+        sleep 0.1
+        @server.close
+        @loop.stop if @running
+        @mutex_hash.stop
+        @thread.join if @thread
+        @log.debug("calling stop in counter server #{@addr}:#{@port}")
+      end
+
+      def on_message(data)
+        errors = Validator.request(data)
+        unless errors.empty?
+          return { 'id' => data['id'], 'data' => [], 'errors' => errors }
+        end
+
+        result = safe_run do
+          send(data['method'], data['params'], data['scope'], data['options'])
+        end
+        result.merge('id' => data['id'])
+      rescue => e
+        @log.error e.to_s
+      end
+
+      private
+
+      def establish(params, _scope, _options)
+        validator = Fluent::Counter::ArrayValidator.new(:empty, :scope)
+        valid_params, errors = validator.call(params)
+        res = Response.new(errors)
+
+        if scope = valid_params.first
+          new_scope = "#{@name}\t#{scope}"
+          res.push_data new_scope
+          @log.debug("Establish new key: #{new_scope}")
+        end
+
+        res.to_hash
+      end
+
+      def init(params, scope, options)
+        validator = Fluent::Counter::HashValidator.new(:empty, :name, :reset_interval)
+        valid_params, errors = validator.call(params)
+        res = Response.new(errors)
+        key_hash = valid_params.reduce({}) do |acc, vp|
+          acc.merge(Store.gen_key(scope, vp['name']) => vp)
+        end
+
+        do_init = lambda do |store, key|
+          begin
+            param = key_hash[key]
+            v = store.init(key, param, ignore: options['ignore'])
+            @log.debug("Create new key: #{param['name']}")
+            res.push_data v
+          rescue => e
+            res.push_error e
+          end
+        end
+
+        if options['random']
+          @mutex_hash.synchronize_keys(*(key_hash.keys), &do_init)
+        else
+          @mutex_hash.synchronize(*(key_hash.keys), &do_init)
+        end
+
+        res.to_hash
+      end
+
+      def delete(params, scope, options)
+        validator = Fluent::Counter::ArrayValidator.new(:empty, :key)
+        valid_params, errors = validator.call(params)
+        res = Response.new(errors)
+        keys = valid_params.map { |vp| Store.gen_key(scope, vp) }
+
+        do_delete = lambda do |store, key|
+          begin
+            v = store.delete(key)
+            @log.debug("delete a key: #{key}")
+            res.push_data v
+          rescue => e
+            res.push_error e
+          end
+        end
+
+        if options['random']
+          @mutex_hash.synchronize_keys(*keys, &do_delete)
+        else
+          @mutex_hash.synchronize(*keys, &do_delete)
+        end
+
+        res.to_hash
+      end
+
+      def inc(params, scope, options)
+        validate_param = [:empty, :name, :value]
+        validate_param << :reset_interval if options['force']
+        validator = Fluent::Counter::HashValidator.new(*validate_param)
+        valid_params, errors = validator.call(params)
+        res = Response.new(errors)
+        key_hash = valid_params.reduce({}) do |acc, vp|
+          acc.merge(Store.gen_key(scope, vp['name']) => vp)
+        end
+
+        do_inc = lambda do |store, key|
+          begin
+            param = key_hash[key]
+            v = store.inc(key, param, force: options['force'])
+            @log.debug("Increment #{key} by #{param['value']}")
+            res.push_data v
+          rescue => e
+            res.push_error e
+          end
+        end
+
+        if options['random']
+          @mutex_hash.synchronize_keys(*(key_hash.keys), &do_inc)
+        else
+          @mutex_hash.synchronize(*(key_hash.keys), &do_inc)
+        end
+
+        res.to_hash
+      end
+
+      def reset(params, scope, options)
+        validator = Fluent::Counter::ArrayValidator.new(:empty, :key)
+        valid_params, errors = validator.call(params)
+        res = Response.new(errors)
+        keys = valid_params.map { |vp| Store.gen_key(scope, vp) }
+
+        do_reset = lambda do |store, key|
+          begin
+            v = store.reset(key)
+            @log.debug("Reset #{key}'s' counter value")
+            res.push_data v
+          rescue => e
+            res.push_error e
+          end
+        end
+
+        if options['random']
+          @mutex_hash.synchronize_keys(*keys, &do_reset)
+        else
+          @mutex_hash.synchronize(*keys, &do_reset)
+        end
+
+        res.to_hash
+      end
+
+      def get(params, scope, _options)
+        validator = Fluent::Counter::ArrayValidator.new(:empty, :key)
+        valid_params, errors = validator.call(params)
+        res = Response.new(errors)
+
+        keys = valid_params.map { |vp| Store.gen_key(scope, vp) }
+        keys.each do |key|
+          begin
+            v = @store.get(key, raise_error: true)
+            @log.debug("Get counter value: #{key}")
+            res.push_data v
+          rescue => e
+            res.push_error e
+          end
+        end
+        res.to_hash
+      end
+
+      def safe_run
+        yield
+      rescue => e
+        {
+          'errors' => [InternalServerError.new(e).to_hash],
+          'data' => []
+        }
+      end
+
+      class Response
+        def initialize(errors = [], data = [])
+          @errors = errors
+          @data = data
+        end
+
+        def push_error(error)
+          @errors << error
+        end
+
+        def push_data(data)
+          @data << data
+        end
+
+        def to_hash
+          if @errors.empty?
+            { 'data' => @data }
+          else
+            errors = @errors.map do |e|
+              error = e.respond_to?(:to_hash) ? e : InternalServerError.new(e.to_s)
+              error.to_hash
+            end
+            { 'data' => @data, 'errors' => errors }
+          end
+        end
+      end
+    end
+
+    class Handler < Fluent::Counter::BaseSocket
+      def initialize(io, on_message)
+        super(io)
+        @on_message = on_message
+      end
+
+      def on_message(data)
+        res = @on_message.call(data)
+        packed_write res if res
+      end
+    end
+  end
+end

data/lib/fluent/counter/store.rb

@@ -0,0 +1,205 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config'
+require 'fluent/counter/error'
+require 'fluent/plugin/storage_local'
+require 'fluent/time'
+
+module Fluent
+  module Counter
+    class Store
+      def self.gen_key(scope, key)
+        "#{scope}\t#{key}"
+      end
+
+      def initialize(opt = {})
+        @log = opt[:log] || $log
+
+        # Notice: This storage is not be implemented auto save.
+        @storage = Plugin.new_storage('local', parent: DummyParent.new(@log))
+        conf = if opt[:path]
+                 {'persistent' => true, 'path' => opt[:path] }
+               else
+                 {'persistent' => false }
+               end
+        @storage.configure(Fluent::Config::Element.new('storage', {}, conf, []))
+      end
+
+      # This class behaves as a configurable plugin for using in storage (OwnedByMixin).
+      class DummyParent
+        include Configurable
+
+        attr_reader :log
+
+        def initialize(log)
+          @log = log
+        end
+
+        def plugin_id
+          'dummy_parent_store'
+        end
+
+        def plugin_id_configured?
+          false
+        end
+
+        # storage_local calls PluginId#plugin_root_dir
+        def plugin_root_dir
+          nil
+        end
+      end
+
+      def start
+        @storage.load
+      end
+
+      def stop
+        @storage.save
+      end
+
+      def init(key, data, ignore: false)
+        ret = if v = get(key)
+                raise InvalidParams.new("#{key} already exists in counter") unless ignore
+                v
+              else
+                @storage.put(key, build_value(data))
+              end
+
+        build_response(ret)
+      end
+
+      def get(key, raise_error: false, raw: false)
+        ret = if raise_error
+                @storage.get(key) or raise UnknownKey.new("`#{key}` doesn't exist in counter")
+              else
+                @storage.get(key)
+              end
+        if raw
+          ret
+        else
+          ret && build_response(ret)
+        end
+      end
+
+      def key?(key)
+        !!@storage.get(key)
+      end
+
+      def delete(key)
+        ret = @storage.delete(key) or raise UnknownKey.new("`#{key}` doesn't exist in counter")
+        build_response(ret)
+      end
+
+      def inc(key, data, force: false)
+        value = data.delete('value')
+        init(key, data) if !key?(key) && force
+        v = get(key, raise_error: true, raw: true)
+        valid_type!(v, value)
+
+        v['total'] += value
+        v['current'] += value
+        t = EventTime.now
+        v['last_modified_at'] = [t.sec, t.nsec]
+        @storage.put(key, v)
+
+        build_response(v)
+      end
+
+      def reset(key)
+        v = get(key, raise_error: true, raw: true)
+        success = false
+        old_data = v.dup
+        now = EventTime.now
+        last_reset_at = EventTime.new(*v['last_reset_at'])
+
+        #  Does it need reset?
+        if (last_reset_at + v['reset_interval']) <= now
+          success = true
+          v['current'] = initial_value(v['type'])
+          t = [now.sec, now.nsec]
+          v['last_reset_at'] = t
+          v['last_modified_at'] = t
+          @storage.put(key, v)
+        end
+
+        {
+          'elapsed_time' => now - last_reset_at,
+          'success' => success,
+          'counter_data' => build_response(old_data)
+        }
+      end
+
+      private
+
+      def build_response(d)
+        {
+          'name' => d['name'],
+          'total' => d['total'],
+          'current' => d['current'],
+          'type' => d['type'],
+          'reset_interval' => d['reset_interval'],
+          'last_reset_at' => EventTime.new(*d['last_reset_at']),
+        }
+      end
+
+      # value is Hash. value requires these fileds.
+      # :name, :total, :current, :type, :reset_interval, :last_reset_at, :last_modified_at
+      def build_value(data)
+        type = data['type'] || 'numeric'
+        now = EventTime.now
+        t = [now.sec, now.nsec]
+
+        v = initial_value(type)
+
+        data.merge(
+          'type' => type,
+          'last_reset_at' => t,
+          'last_modified_at' => t,
+          'current' => v,
+          'total' => v,
+        )
+      end
+
+      def initial_value(type)
+        case type
+        when 'numeric', 'integer' then 0
+        when 'float' then 0.0
+        else raise InvalidParams.new('`type` should be integer, float, or numeric')
+        end
+      end
+
+      def valid_type!(v, value)
+        type = v['type']
+        return unless (type != 'numeric') && (type_str(value) != type)
+        raise InvalidParams.new("`type` is #{type}. You should pass #{type} value as a `value`")
+      end
+
+      def type_str(v)
+        case v
+        when Integer
+          'integer'
+        when Float
+          'float'
+        when Numeric
+          'numeric'
+        else
+          raise InvalidParams.new("`type` should be integer, float, or numeric")
+        end
+      end
+    end
+  end
+end

data/lib/fluent/counter/validator.rb

@@ -0,0 +1,145 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/counter/error'
+
+module Fluent
+  module Counter
+    class Validator
+      VALID_NAME = /\A[a-z][a-zA-Z0-9\-_]*\Z/
+      VALID_SCOPE_NAME = /\A[a-z][\ta-zA-Z0-9\-_]*\Z/
+      VALID_METHODS = %w(establish init delete inc get reset)
+
+      def self.request(data)
+        errors = []
+        raise "Received data is not Hash: #{data}" unless data.is_a?(Hash)
+
+        unless data['id']
+          errors << Fluent::Counter::InvalidRequest.new('Request should include `id`')
+        end
+
+        if !data['method']
+          errors << Fluent::Counter::InvalidRequest.new('Request should include `method`')
+        elsif !(VALID_NAME =~ data['method'])
+          errors << Fluent::Counter::InvalidRequest.new('`method` is the invalid format')
+        elsif !VALID_METHODS.include?(data['method'])
+          errors << Fluent::Counter::MethodNotFound.new("Unknown method name passed: #{data['method']}")
+        end
+
+        errors.map(&:to_hash)
+      end
+
+      def initialize(*types)
+        @types = types.map(&:to_s)
+        @empty = @types.delete('empty')
+      end
+
+      def call(data)
+        success = []
+        errors = []
+
+        if @empty && data.empty?
+          errors << Fluent::Counter::InvalidParams.new('One or more `params` are required')
+        else
+          data.each do |d|
+            begin
+              @types.each { |type| dispatch(type, d) }
+              success << d
+            rescue => e
+              errors << e
+            end
+          end
+        end
+
+        [success, errors]
+      end
+
+      private
+
+      def dispatch(type, data)
+        send("validate_#{type}!", data)
+      rescue NoMethodError => e
+        raise Fluent::Counter::InternalServerError.new(e)
+      end
+    end
+
+    class ArrayValidator < Validator
+      def validate_key!(name)
+        unless name.is_a?(String)
+          raise Fluent::Counter::InvalidParams.new('The type of `key` should be String')
+        end
+
+        unless VALID_NAME =~ name
+          raise Fluent::Counter::InvalidParams.new('`key` is the invalid format')
+        end
+      end
+
+      def validate_scope!(name)
+        unless name.is_a?(String)
+          raise Fluent::Counter::InvalidParams.new('The type of `scope` should be String')
+        end
+
+        unless VALID_SCOPE_NAME =~ name
+          raise Fluent::Counter::InvalidParams.new('`scope` is the invalid format')
+        end
+      end
+    end
+
+    class HashValidator < Validator
+      def validate_name!(hash)
+        name = hash['name']
+        unless name
+          raise Fluent::Counter::InvalidParams.new('`name` is required')
+        end
+
+        unless name.is_a?(String)
+          raise Fluent::Counter::InvalidParams.new('The type of `name` should be String')
+        end
+
+        unless VALID_NAME =~ name
+          raise Fluent::Counter::InvalidParams.new("`name` is the invalid format")
+        end
+      end
+
+      def validate_value!(hash)
+        value = hash['value']
+        unless value
+          raise Fluent::Counter::InvalidParams.new('`value` is required')
+        end
+
+        unless value.is_a?(Numeric)
+          raise Fluent::Counter::InvalidParams.new("The type of `value` type should be Numeric")
+        end
+      end
+
+      def validate_reset_interval!(hash)
+        interval = hash['reset_interval']
+
+        unless interval
+          raise Fluent::Counter::InvalidParams.new('`reset_interval` is required')
+        end
+
+        unless interval.is_a?(Numeric)
+          raise Fluent::Counter::InvalidParams.new('The type of `reset_interval` should be Numeric')
+        end
+
+        if interval < 0
+          raise Fluent::Counter::InvalidParams.new('`reset_interval` should be a positive number')
+        end
+      end
+    end
+  end
+end

data/lib/fluent/counter.rb

@@ -0,0 +1,23 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/counter/client'
+require 'fluent/counter/server'
+
+module Fluent
+  module Counter
+  end
+end

data/lib/fluent/daemon.rb

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+here = File.dirname(__FILE__)
+$LOAD_PATH << File.expand_path(File.join(here, '..'))
+
+require 'serverengine'
+require 'fluent/supervisor'
+
+server_module = Fluent.const_get(ARGV[0])
+worker_module = Fluent.const_get(ARGV[1])
+# it doesn't call ARGV in block because when reloading config, params will be initialized and then it can't use previous config.
+config_path = ARGV[2]
+params = JSON.parse(ARGV[3])
+ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.load_config(config_path, params) }