fluent-plugin-pghstore 0.0.1

data/.gitignore

@@ -0,0 +1,11 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+# For TextMate, emacs, vim
+*.tmproj
+tmtags
+*~
+\#*
+.\#*
+*.swp

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in fluent-plugin-pghstore.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2012- Shirou WAKAYAMA
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.rdoc

@@ -0,0 +1,76 @@
+= fluent-plugin-pghstore
+
+== Component
+
+=== PgHStoreOutput
+
+Output to PostgreSQL hstore database. 
+
+Output table should have tag, time and record column.:
+
+  CREATE TABLE #{tablename} (
+    tag TEXT,
+    time TIMESTAMP WITH TIME ZONE,
+    record HSTORE
+  );
+
+== Requirement
+
+- PostgreSQL 9.0 or higher
+  - hstore changed at PostgreSQL 9.0. 
+- postgres-contrib
+- ruby-pg
+
+=== How to Install hstore
+
+hstore is in the contrib. 
+
+9.1 or higher:
+
+  psql <dbname> -c "CREATE EXTENSION hstore;"
+
+9.0:
+
+  psql <dbname> -f ${PGHOME}/share/contrib/hstore.sql
+
+== Configuration
+
+Example:
+
+  <match apache.*>
+    type pghstore
+    database test
+    table test 
+    table_option CREATE INDEX time_index ON testb (time);
+  </match>
+
+=== Options
+
+- Required
+  - database 
+    - database name
+- Optional
+  - table
+    - tablename. If not set, use +fluentd_store+. If not exists, creates automatically.
+  - host
+  - port
+  - user
+  - password
+  - table_option 
+    - Add some SQL. This SQL is called only once when table is created from this plugin.
+
+== Limitation
+
+- Nested output is not allowd.
+- Since using only one connection, performance may become bad. When you meet this, use connection pooling and write patch!
+
+
+== Thanks
+
+This source code is mainly borrowed from
+{fluent-plugin-datacounter}{https://rubygems.org/gems/fluent-plugin-datacounter}. Thank you for tagomoris.
+
+== Copyright
+
+Copyright:: Copyright (c) 2012- Shirou WAKAYAMA
+License::   Apache License

data/Rakefile

@@ -0,0 +1,12 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+#  test.pattern = 'test/**/test_*.rb'
+   test.test_files = FileList['test/**/test*.rb']
+  test.verbose = true
+end
+
+task :default => :test
+

data/example.conf

@@ -0,0 +1,13 @@
+<source>
+  type tail
+  path /var/log/apache/access_log
+  tag apache.access
+  format apache
+</source>
+
+<match apache.*>
+  type pghstore
+  database test
+  table test  # (option) default is fluentd_store
+  table_option CREATE INDEX time_index ON testb (time);  # (option)
+</match>

data/fluent-plugin-pghstore.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "fluent-plugin-pghstore"
+  s.version     = "0.0.1"
+  s.authors     = ["WAKAYAMA Shirou"]
+  s.email       = ["shirou.faw@gmail.com"]
+  s.homepage    = "https://github.com/r_rudi/fluent-plugin-pghstore"
+  s.summary     = %q{Output to PostgreSQL database which has a hstore extension}
+  s.description = %q{Output to PostgreSQL database which has a hstore extension}
+
+  s.rubyforge_project = "fluent-plugin-pghstore"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+  s.add_development_dependency "fluentd"
+  s.add_development_dependency "pg"
+  s.add_runtime_dependency "fluentd"
+  s.add_runtime_dependency "pg"
+end

data/lib/fluent/plugin/out_pghstore.rb

@@ -0,0 +1,102 @@
+class Fluent::PgHStoreOutput < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('pghstore', self)
+
+  config_param :database, :string
+  config_param :table, :string, :default => 'fluentd_store'
+  config_param :host, :string, :default => 'localhost'
+  config_param :port, :integer, :default => 5432
+  config_param :user, :string, :default => nil
+  config_param :password, :string, :default => nil
+
+  config_param :table_option, :string, :default => nil
+
+  def initialize
+    super
+    require 'pg'
+  end
+
+  def start
+    super
+
+    @conn = get_connection(@database, @host, @port, @user, @password)
+
+    create_table(@table) unless table_exists?(@table)
+
+  end
+
+  def shutdown
+    super
+
+    @conn.close
+  end
+  
+  def format(tag, time, record)
+    [tag, time, record].to_msgpack
+  end
+
+  def write(chunk)
+    chunk.msgpack_each {|(tag, time_str, record)|
+      sql = generate_sql(tag, time_str, record)
+      @conn.exec(sql)
+    }
+  end
+
+  private
+
+  def generate_sql(tag, time, record)
+    kv_list = []
+    record.each {|(key,value)|
+      begin
+        v = Integer(value)
+      rescue ArgumentError => e
+        kv_list.push("#{key} => \"#{value}\"")  # might be string
+      else
+        kv_list.push("#{key} => #{value}")
+      end
+    }
+
+    sql =<<"SQL"
+INSERT INTO #{@table} (tag, time, record) VALUES
+('#{tag}', '#{Time.at(time)}'::TIMESTAMP WITH TIME ZONE, '#{kv_list.join(",")}');
+SQL
+    return sql
+  end
+
+  def get_connection(dbname, host, port, user, password)
+    if user
+      return PG.connect(:dbname => dbname, :host => host, :port => port,
+                        :user => user, :password => password)
+    else
+      return PG.connect(:dbname => dbname, :host => host, :port => port)
+    end
+  end
+
+  def table_exists?(table)
+    sql =<<"SQL"
+SELECT COUNT(*) FROM pg_tables WHERE tablename = '#{table}';
+SQL
+    res = @conn.exec(sql)
+    if res[0]["count"] == "1"
+      return true
+    else
+      return false
+    end
+  end
+
+  def create_table(tablename)
+    sql =<<"SQL"
+CREATE TABLE #{tablename} (
+  tag TEXT,
+  time TIMESTAMP WITH TIME ZONE,
+  record HSTORE
+);
+SQL
+
+    sql += @table_option if @table_option
+
+    @conn.exec(sql)
+
+    $log.warn "#{tablename} table is not exists. created."
+  end
+
+end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_pghstore'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_pghstore.rb

@@ -0,0 +1,44 @@
+require 'helper'
+
+class PGHStoreOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    database "testdb"
+    table "testtable"
+    user "testuser"
+    password "testpassword"
+  ]
+
+  def create_driver(conf = CONFIG, tag='test.input')
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::PgHStoreOutput, tag).configure(conf)
+  end
+
+  def test_configure
+  end
+
+  def test_format
+    d = create_driver
+
+    # time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    # d.emit({"a"=>1}, time)
+    # d.emit({"a"=>2}, time)
+
+    # d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
+    # d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+
+    # d.run
+  end
+
+  def test_write
+    d = create_driver
+
+    # time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    # d.emit({"a"=>1}, time)
+    # d.emit({"a"=>2}, time)
+
+  end
+
+end

data/test/plugin/test_out_pghstore.rb.bak

@@ -0,0 +1,277 @@
+require 'helper'
+
+class DataCounterOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    unit minute
+    aggregate tag
+    input_tag_remove_prefix test
+    count_key target
+    pattern1 status2xx ^2\\d\\d$
+    pattern2 status3xx ^3\\d\\d$
+    pattern3 status4xx ^4\\d\\d$
+    pattern4 status5xx ^5\\d\\d$
+  ]
+
+  def create_driver(conf = CONFIG, tag='test.input')
+    Fluent::Test::OutputTestDriver.new(Fluent::DataCounterOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        count_key field
+      ]
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        pattern1 hoge ^1\\d\\d$
+      ]
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        count_key field
+        pattern2 hoge ^1\\d\\d$
+      ]
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        count_key field
+        pattern1 hoge ^1\\d\\d$
+        pattern4 pos  ^4\\d\\d$
+      ]
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        count_key field
+        pattern1 hoge ^1\\d\\d$
+        pattern2 hoge ^4\\d\\d$
+      ]
+    }
+    d = create_driver %[
+      count_key field
+      pattern1 ok ^2\\d\\d$
+    ]
+    assert_equal 60, d.instance.tick
+    assert_equal :tag, d.instance.aggregate
+    assert_equal 'datacount', d.instance.tag
+    assert_nil d.instance.input_tag_remove_prefix
+    assert_equal 'field', d.instance.count_key
+    assert_equal 'ok ^2\d\d$', d.instance.pattern1
+
+    d1 = create_driver %[
+      unit minute
+      count_key field
+      pattern1 ok ^2\\d\\d$
+    ]
+    d2 = create_driver %[
+      count_interval 60s
+      count_key field
+      pattern1 ok ^2\\d\\d$
+    ]
+    assert_equal d1.instance.tick, d2.instance.tick
+    
+    d = create_driver %[
+      count_interval 5m
+      count_key field
+      pattern1 ok ^2\\d\\d$
+    ]
+    assert_equal 300, d.instance.tick
+
+    d = create_driver %[
+      count_interval 2h
+      count_key field
+      pattern1 ok ^2\\d\\d$
+    ]
+    assert_equal 7200, d.instance.tick
+
+    d = create_driver %[
+      count_interval 30s
+      count_key field
+      pattern1 ok ^2\\d\\d$
+    ]
+    assert_equal 30, d.instance.tick
+  end
+
+  def test_count_initialized
+    d = create_driver %[
+      aggregate all
+      count_key field
+      pattern1 hoge 1\d\d
+      pattern2 moge 2\d\d
+    ]
+    assert_equal [0,0,0], d.instance.counts['all']
+  end
+
+  def test_countups
+    d = create_driver
+    assert_nil d.instance.counts['test.input']
+
+    d.instance.countups('test.input', [0, 0, 0, 0, 0])
+    assert_equal [0,0,0,0,0], d.instance.counts['test.input']
+    d.instance.countups('test.input', [1, 1, 1, 0, 0])
+    assert_equal [1,1,1,0,0], d.instance.counts['test.input']
+    d.instance.countups('test.input', [0, 5, 1, 0, 0])
+    assert_equal [1,6,2,0,0], d.instance.counts['test.input']
+  end
+
+  def test_stripped_tag
+    d = create_driver
+    assert_equal 'input', d.instance.stripped_tag('test.input')
+    assert_equal 'test.input', d.instance.stripped_tag('test.test.input')
+    assert_equal 'input', d.instance.stripped_tag('input')
+  end
+
+  def test_generate_output
+    d = create_driver
+    r1 = d.instance.generate_output({'test.input' => [60,240,120,180,0], 'test.input2' => [0,600,0,0,0]}, 60)
+    assert_equal   60, r1['input_unmatched_count']
+    assert_equal  1.0, r1['input_unmatched_rate']
+    assert_equal 10.0, r1['input_unmatched_percentage']
+    assert_equal  240, r1['input_status2xx_count']
+    assert_equal  4.0, r1['input_status2xx_rate']
+    assert_equal 40.0, r1['input_status2xx_percentage']
+    assert_equal  120, r1['input_status3xx_count']
+    assert_equal  2.0, r1['input_status3xx_rate']
+    assert_equal 20.0, r1['input_status3xx_percentage']
+    assert_equal  180, r1['input_status4xx_count']
+    assert_equal  3.0, r1['input_status4xx_rate']
+    assert_equal 30.0, r1['input_status4xx_percentage']
+    assert_equal    0, r1['input_status5xx_count']
+    assert_equal  0.0, r1['input_status5xx_rate']
+    assert_equal  0.0, r1['input_status5xx_percentage']
+
+    assert_equal    0, r1['input2_unmatched_count']
+    assert_equal  0.0, r1['input2_unmatched_rate']
+    assert_equal  0.0, r1['input2_unmatched_percentage']
+    assert_equal  600, r1['input2_status2xx_count']
+    assert_equal 10.0, r1['input2_status2xx_rate']
+    assert_equal 100.0, r1['input2_status2xx_percentage']
+    assert_equal    0, r1['input2_status3xx_count']
+    assert_equal  0.0, r1['input2_status3xx_rate']
+    assert_equal  0.0, r1['input2_status3xx_percentage']
+    assert_equal    0, r1['input2_status4xx_count']
+    assert_equal  0.0, r1['input2_status4xx_rate']
+    assert_equal  0.0, r1['input2_status4xx_percentage']
+    assert_equal    0, r1['input2_status5xx_count']
+    assert_equal  0.0, r1['input2_status5xx_rate']
+    assert_equal  0.0, r1['input2_status5xx_percentage']
+
+    d = create_driver %[
+      aggregate all
+      count_key field
+      pattern1 hoge xxx\d\d
+    ]
+    r2 = d.instance.generate_output({'all' => [60,240]}, 60)
+    assert_equal   60, r2['unmatched_count']
+    assert_equal  1.0, r2['unmatched_rate']
+    assert_equal 20.0, r2['unmatched_percentage']
+    assert_equal  240, r2['hoge_count']
+    assert_equal  4.0, r2['hoge_rate']
+    assert_equal 80.0, r2['hoge_percentage']
+  end
+
+  def test_pattern_num
+    assert_equal 20, Fluent::DataCounterOutput::PATTERN_MAX_NUM
+
+    conf = %[
+      aggregate all
+      count_key field
+    ]
+    (1..20).each do |i|
+      conf += "pattern#{i} name#{i} ^#{i}$\n"
+    end
+    d = create_driver(conf, 'test.max')
+    d.run do
+      (1..20).each do |j|
+        d.emit({'field' => j})
+      end
+    end
+    r = d.instance.flush(60)
+    assert_equal 1, r['name1_count']
+    assert_equal 1, r['name2_count']
+    assert_equal 1, r['name3_count']
+    assert_equal 1, r['name4_count']
+    assert_equal 1, r['name5_count']
+    assert_equal 1, r['name6_count']
+    assert_equal 1, r['name7_count']
+    assert_equal 1, r['name8_count']
+    assert_equal 1, r['name9_count']
+    assert_equal 1, r['name10_count']
+    assert_equal 1, r['name11_count']
+    assert_equal 1, r['name12_count']
+    assert_equal 1, r['name13_count']
+    assert_equal 1, r['name14_count']
+    assert_equal 1, r['name15_count']
+    assert_equal 1, r['name16_count']
+    assert_equal 1, r['name17_count']
+    assert_equal 1, r['name18_count']
+    assert_equal 1, r['name19_count']
+    assert_equal 1, r['name20_count']
+  end
+
+  def test_emit
+    d1 = create_driver(CONFIG, 'test.tag1')
+    d1.run do
+      60.times do
+        d1.emit({'target' => '200'})
+        d1.emit({'target' => '100'})
+        d1.emit({'target' => '200'})
+        d1.emit({'target' => '400'})
+      end
+    end
+    r1 = d1.instance.flush(60)
+    assert_equal 120, r1['tag1_status2xx_count']
+    assert_equal 2.0, r1['tag1_status2xx_rate']
+    assert_equal 50.0, r1['tag1_status2xx_percentage']
+
+    assert_equal 60, r1['tag1_status4xx_count']
+    assert_equal 1.0, r1['tag1_status4xx_rate']
+    assert_equal 25.0, r1['tag1_status4xx_percentage']
+    
+    assert_equal 60, r1['tag1_unmatched_count']
+    assert_equal 1.0, r1['tag1_unmatched_rate']
+    assert_equal 25.0, r1['tag1_unmatched_percentage']
+    
+    assert_equal 0, r1['tag1_status3xx_count']
+    assert_equal 0.0, r1['tag1_status3xx_rate']
+    assert_equal 0.0, r1['tag1_status3xx_percentage']
+    assert_equal 0, r1['tag1_status5xx_count']
+    assert_equal 0.0, r1['tag1_status5xx_rate']
+    assert_equal 0.0, r1['tag1_status5xx_percentage']
+
+    d2 = create_driver(%[
+      aggregate all
+      count_key target
+      pattern1 ok 2\\d\\d
+      pattern2 redirect 3\\d\\d
+    ], 'test.tag2')
+    d2.run do
+      60.times do
+        d2.emit({'target' => '200'})
+        d2.emit({'target' => '300 200'})
+      end
+    end
+    d2.instance.flush_emit(120)
+    emits = d2.emits
+    assert_equal 1, emits.length
+    data = emits[0]
+    assert_equal 'datacount', data[0] # tag
+    assert_equal 120, data[2]['ok_count']
+    assert_equal 1.0, data[2]['ok_rate']
+    assert_equal 100.0, data[2]['ok_percentage']
+    assert_equal 0, data[2]['redirect_count']
+    assert_equal 0.0, data[2]['redirect_rate']
+    assert_equal 0.0, data[2]['redirect_percentage']
+    assert_equal 0, data[2]['unmatched_count']
+    assert_equal 0.0, data[2]['unmatched_rate']
+    assert_equal 0.0, data[2]['unmatched_percentage']
+  end
+
+end

metadata

@@ -0,0 +1,136 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-pghstore
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- WAKAYAMA Shirou
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Output to PostgreSQL database which has a hstore extension
+email:
+- shirou.faw@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- example.conf
+- fluent-plugin-pghstore.gemspec
+- lib/fluent/plugin/out_pghstore.rb
+- test/helper.rb
+- test/plugin/test_out_pghstore.rb
+- test/plugin/test_out_pghstore.rb.bak
+homepage: https://github.com/r_rudi/fluent-plugin-pghstore
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: fluent-plugin-pghstore
+rubygems_version: 1.8.21
+signing_key: 
+specification_version: 3
+summary: Output to PostgreSQL database which has a hstore extension
+test_files: []