fluent-plugin-onlineuser 0.0.1

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+.idea

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-onlineuser.gemspec
+gemspec

data/README.md

@@ -0,0 +1,2 @@
+fluent-plugin-onlineuser
+========================

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-onlineuser.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |gem|
+  gem.name = "fluent-plugin-onlineuser"
+  gem.version = "0.0.1"
+  gem.authors = ["Yuyang Lan"]
+  gem.email = ["lanyuyang@gree.net"]
+  gem.summary = %q{Online Users Counter}
+  gem.description = %q{Fluentd plugin to count online users. It's based on Redis and the sorted set data type.}
+  gem.homepage = "https://github.com/lanyuyang/fluent-plugin-onlineuser"
+
+  gem.files = `git ls-files`.split($\)
+  gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
+  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.has_rdoc = false
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "fluentd"
+  gem.add_development_dependency "redis"
+
+  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "redis"
+end

data/lib/fluent/plugin/out_onlineuser.rb

@@ -0,0 +1,186 @@
+# -*- coding: utf-8 -*-
+
+module Fluent
+  class OnlineuserOutput < BufferedOutput
+    Fluent::Plugin.register_output('online_user', self)
+
+    config_param :host, :string, :default => 'localhost'
+    config_param :port, :integer, :default => 6379
+    config_param :db_index, :integer, :default => 0
+    config_param :redis_retry, :integer, :default => 3
+
+    config_param :session_timeout, :integer, :default => 1800
+
+    config_param :redis_key_prefix, :string, :default => 'counter:online_user:'
+    config_param :user_identify, :string, :default => 'uid'
+    config_param :segment, :string, :default => nil
+
+    config_param :tag, :string, :default => "online_user"
+    config_param :silent, :bool, :default => false
+
+    def initialize
+      super
+      require 'redis'
+      require 'msgpack'
+      require 'rubygems'
+    end
+
+    def configure(conf)
+      super
+      @host = conf.has_key?('host') ? conf['host'] : 'localhost'
+      @port = conf.has_key?('port') ? conf['port'].to_i : 6379
+      @db_number = conf.has_key?('db_number') ? conf['db_number'].to_i : nil
+
+      @user_identify = @user_identify.split '|'
+
+      if @segment
+        s = @segment.split /\s+/
+        case s.first
+          when 'tag'
+            @cal_segment = lambda { |tag, record| 'tag:' + tag }
+          when 'capture'
+            if s.length == 3
+              format = s[2]
+              if format[0] == ?/ and format[-1] == ?/
+                format = format[1..-2]
+              end
+              regexp = Regexp.new(format)
+              @cal_segment = lambda { |tag, record|
+                if record[s[1]] and record[s[1]].is_a? String
+                  if c = record[s[1]].match(regexp)
+                    'captured_' + s[1] + ':' + (c.captures.join '_')
+                  end
+                end
+              }
+            else
+              raise Fluent::ConfigError, "wrong segment capture, specify it like 'capture __FIELD_NAME__ __CAPTURE_REGEX__'"
+            end
+          else
+            @cal_segment = lambda { |tag, record|
+              if record[@segment]
+                @segment + ':' + record[@segment]
+              end
+            }
+        end
+      end
+    end
+
+    def start
+      super
+
+      begin
+        gem "hiredis"
+        @redis = Redis.new(
+            :host => @host, :port => @port, :driver => :hiredis,
+            :thread_safe => true, :db => @db_index
+        )
+      rescue LoadError
+        @redis = Redis.new(
+            :host => @host, :port => @port,
+            :thread_safe => true, :db => @db_index
+        )
+      end
+
+      @segments = {}
+
+      start_watch
+    end
+
+    def shutdown
+      @redis.quit
+    end
+
+    def to_redis_key(segment_str)
+      @redis_key_prefix + segment_str
+    end
+
+    def tryOnRedis(method, *args)
+      tries = 0
+      begin
+        @redis.send(method, *args) if @redis.respond_to? method
+      rescue Redis::CommandError => e
+        tries += 1
+        # retry 3 times
+        retry if tries <= @redis_retry
+        $log.warn %Q[redis command retry failed : #{method}(#{args.join(', ')})]
+        raise e.message
+      end
+    end
+
+    def start_watch
+      @watcher = Thread.new(&method(:watch))
+    end
+
+    def watch
+      @last_checked = Fluent::Engine.now
+      tick = 60
+      while true
+        sleep 0.5
+        if Fluent::Engine.now - @last_checked >= tick
+          now = Fluent::Engine.now
+
+          @segments.each_key do |segment|
+            user_num = user_count segment
+
+            if user_num == 0
+              @segments.delete segment
+            elsif not @silent
+              Fluent::Engine.emit @tag + '.' + segment, Fluent::Engine.now, {"online_user" => user_num}
+            end
+          end
+
+          @last_checked = now
+        end
+      end
+    end
+
+    def user_count(segment)
+      to_expire = Fluent::Engine.now - @session_timeout
+      key = to_redis_key segment
+      tryOnRedis 'zremrangebyscore', key, '-inf', to_expire
+      tryOnRedis 'zcard', key
+    end
+
+    def extract_uid(record)
+      i = @user_identify.detect { |id|
+        v = record[id]
+        (v.is_a?(String) and !v.empty?) or (v.is_a?(Numeric) and v != 0)
+      }
+      i ? record[i] : nil
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+
+      online_user = Hash.new { |hash, key| hash[key] = Hash.new }
+
+      begin
+        chunk.msgpack_each do |(tag, time, record)|
+
+          if (uid = extract_uid record)
+            online_user['all'][uid] = time
+
+            if @cal_segment and @cal_segment != '' and segment = @cal_segment.call(tag, record)
+              online_user[segment][uid] = time
+            end
+          end
+        end
+
+        ## write to redis
+        online_user.each { |segment, users|
+          @segments[segment] = true
+
+          key = to_redis_key segment
+
+          users.each { |uid, ts|
+            tryOnRedis 'zadd', key, ts, uid
+          }
+        }
+
+      end
+    end
+  end
+end

data/test/fluentd.conf

@@ -0,0 +1,51 @@
+<source>
+  type forward
+</source>
+
+<match access_log.**>
+  type online_user
+  flush_interval 10s
+
+  #redis configuration
+  #host localhost
+  #port 6379
+  #db_index 0
+  #redis_try 3
+
+  #inactive timeout, default 30 minutes
+  #session_timeout 1800
+
+  # prefix of the keys stored into Redis
+  #redis_key_prefix counter:online_user:
+
+  # how do you define unique user ?
+  # the default is to find 'uid' field in record
+
+  # for support both members and non-members, try:
+  #user_identify uid|ip
+
+  #emit online user count event periodically or not
+  #silent false
+
+  #the tag prefix for emitted event
+  #tag online_user
+
+  #besides counting the overall online users
+  #also wanna count online users for each segment ?
+  #if so, use the segment directive
+
+  #segmented by tag
+  #segment tag
+
+  #segmented by a specified field
+  #segment service
+
+  #segmented by regexp-captured string
+  #the format is: capture __FIELD__ __REGEXP__
+  #segment capture name /^\w+:(\w+):/
+
+</match>
+
+<match **>
+  type stdout
+</match>

data/test/plugin/test_out_onlineuser.rb

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+
+require 'test/unit'
+require 'fluent/test'
+require 'fluent/plugin/out_onlineuser'
+require 'redis'
+
+class OnlineuserOutputTest < Test::Unit::TestCase
+  CONFIG = %[
+      host localhost
+      port 6379
+      db_number 0
+    ]
+
+  @redis
+
+  def setup
+    Fluent::Test.setup
+    @redis = Redis.new(:host => 'localhost', :port => 6379, :thread_safe => true, :db => 0)
+  end
+
+  def teardown
+    @redis.quit
+  end
+
+  def create_driver(conf, tag='test')
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::OnlineuserOutput, tag).configure(conf)
+  end
+
+  def test_event_counter
+    d = create_driver CONFIG
+    t = Time.now
+    time = t.to_i
+    d.emit({"name" => "test", "uid" => 1}, time)
+    d.emit({"name" => "test", "uid" => 2}, time)
+    d.emit({"name" => "test", "uid" => 3}, time)
+    d.run
+
+    assert_equal 3, @redis.zcard('counter:online_user:all')
+  end
+
+end

metadata

@@ -0,0 +1,136 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-onlineuser
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Yuyang Lan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin to count online users. It's based on Redis and the sorted
+  set data type.
+email:
+- lanyuyang@gree.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- fluent-plugin-onlineuser.gemspec
+- lib/fluent/plugin/out_onlineuser.rb
+- test/fluentd.conf
+- test/plugin/test_out_onlineuser.rb
+homepage: https://github.com/lanyuyang/fluent-plugin-onlineuser
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Online Users Counter
+test_files:
+- test/fluentd.conf
+- test/plugin/test_out_onlineuser.rb