fluent-plugin-pgjson 0.0.1

data/.gitignore

@@ -0,0 +1,11 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg
+.DS_Store
+*.tmproj
+tmtags
+*~
+\#*
+.\#*
+*.swp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-pgjson.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,3 @@
+Copyright (c) 2012 OKUNO Akihiro
+
+Apache License, Version 2.0

data/README.md

@@ -0,0 +1,69 @@
+# Fluent::Plugin::PgJson
+
+Output Plugin for PostgreSQL Json Type.
+
+<b>Json type is availble in PostgreSQL version over 9.2</b>
+
+## Installation
+
+`$ fluent-gem install fluent-plugin-pgjson`
+
+## Schema
+
+Specified table must have this schema.
+
+|col|type|
+|---|---|
+|{tag_col}|Text|
+|{time_col}|Timestamp WITH TIME ZONE|
+|{record_col}|Json|
+
+### Example
+
+```
+CREATE TABLE fluentd (
+    tag Text
+    ,time Timestamptz
+    ,record Json
+);
+```
+
+## Configuration
+
+### Example
+
+```
+<match **>
+  type pgjson
+  host localhost
+  port 5432
+  database fluentd
+  table fluentd
+  user postgres
+  password postgres
+  time_col time
+  tag_col tag
+  record_col record
+</match>
+```
+
+### Parameter
+
+|parameter|description|default|
+|---|---|---|
+|host|postgres server hostname|localhost|
+|port|postgres server port number|5432|
+|database|database name to which records will be inserted||
+|table|table name to which records will be inserted||
+|user|user name used to connect database|nil|
+|password|password uset to connect database|nil|
+|time_col|column name to insert time|time|
+|tag_col|column name to insert tag|tag|
+|record_col|column name to insert record|record|
+
+## Copyright
+
+<table>
+<tr><td>Copyright</td><td>Copyright (c) 2012 OKUNO Akihiro</td></tr>
+<tr><td>License</td><td>Apache License, Version 2.0</td></tr>
+</table>

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/example.conf

@@ -0,0 +1,16 @@
+<source>
+  type forward
+</source>
+
+<match **>
+  type pgjson
+  host localhost
+  port 5432
+  database fluentd
+  table fluentd
+  user postgres
+  password postgres
+  time_col time
+  tag_col tag
+  record_col record
+</match>

data/fluent-plugin-pgjson.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "fluent-plugin-pgjson"
+  s.version     = "0.0.1"
+  s.authors     = ["OKUNO Akihiro"]
+  s.email       = ["okuno.akihiro@gmail.com"]
+  s.homepage    = "https://github.com/choplin/fluent-plugin-pgjson"
+  s.summary     = %q{}
+  s.description = %q{}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency "fluentd"
+  s.add_runtime_dependency "pg"
+end

data/lib/fluent/plugin/out_pgjson.rb

@@ -0,0 +1,89 @@
+module Fluent
+
+class PgJsonOutput < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('pgjson', self)
+
+  config_param :host       , :string  , :default => 'localhost'
+  config_param :port       , :integer , :default => 5432
+  config_param :database   , :string
+  config_param :table      , :string
+  config_param :user       , :string  , :default => nil
+  config_param :password   , :string  , :default => nil
+  config_param :time_col   , :string  , :default => 'time'
+  config_param :tag_col    , :string  , :default => 'tag'
+  config_param :record_col , :string  , :default => 'record'
+
+
+  def initialize
+    super
+    require 'pg'
+  end
+
+  def configure(conf)
+    super
+    @stmt_name = 'insert'
+  end
+
+  def start
+    super
+    init_connection
+  end
+
+  def shutdown
+    super
+    if !@conn.nil? and !@conn.finished?
+      @conn.close()
+    end
+  end
+
+  def format(tag, time, record)
+    [tag, time, record].to_msgpack
+  end
+
+  def write(chunk)
+    begin
+      sql = build_sql(chunk)
+      @conn.exec(sql)
+    rescue
+      #TODO
+      raise
+    end
+  end
+
+  private
+  def init_connection
+    begin
+      @conn = PGconn.new(:dbname => @database, :host => @host, :port => @port, :user => @user, :password => @password)
+      @conn.setnonblocking(true)
+    rescue
+      raise Fluent::ConfigError, "failed to connect PostgreSQL Server at #{@host}:#{@port}"
+    end
+  end
+
+  def prepare_statement
+    begin
+      @conn.prepare(@stmt_name, sql)
+    rescue
+      #TODO
+      raise
+    end
+  end
+
+  def build_sql(chunk)
+    values = build_values(chunk)
+    sql =<<"SQL"
+INSERT INTO #{@table} (#{@tag_col}, #{@time_col}, #{@record_col})
+VALUES #{values};
+SQL
+  end
+
+  def build_values(chunk)
+    tmp = []
+    chunk.msgpack_each do |tag, time, record|
+      tmp << ("("+[tag, Time.at(time), record.to_json].map{|s| @conn.escape_literal(s.to_s)}.join(',')+")")
+    end
+    tmp.join(',')
+  end
+end
+
+end

data/test/helper.rb

@@ -0,0 +1,27 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+require 'fluent/plugin/out_pgjson'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out.rb

@@ -0,0 +1,46 @@
+require 'helper'
+
+class PgJsonOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    type pgjson
+    host localhost
+    port 5432
+    database fluentd
+    table fluentd
+    user postgres
+    password postgres
+    time_col time
+    tag_col tag
+    record_col record
+  ]
+
+  def create_driver(conf = CONFIG, tag='test')
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::PgJsonOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+
+    assert_equal "localhost", d.instance.host
+    assert_equal 5432, d.instance.port
+    assert_equal "fluentd", d.instance.database
+    assert_equal "fluentd", d.instance.table
+    assert_equal "postgres", d.instance.user
+    assert_equal "postgres", d.instance.password
+    assert_equal "time", d.instance.time_col
+    assert_equal "tag", d.instance.tag_col
+    assert_equal "record", d.instance.record_col
+  end
+
+  def test_format
+    d = create_driver
+  end
+
+  def test_write
+    d = create_driver
+  end
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-pgjson
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- OKUNO Akihiro
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-07-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- okuno.akihiro@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- example.conf
+- fluent-plugin-pgjson.gemspec
+- lib/fluent/plugin/out_pgjson.rb
+- test/helper.rb
+- test/plugin/test_out.rb
+homepage: https://github.com/choplin/fluent-plugin-pgjson
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: ''
+test_files:
+- test/helper.rb
+- test/plugin/test_out.rb
+has_rdoc: