fluent-plugin-pgjson 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a690b321412f2b0fe7dd2537e38c284b3d6753ee
-  data.tar.gz: abed844423a52e625fc1798231224df083b7e8ba
+  metadata.gz: 4dfa8ba1bbd21e1ffa84476064f2cb0eced7e5c8
+  data.tar.gz: 7522f3e3b4639c1022f58f41ee9b568fea58827a
 SHA512:
-  metadata.gz: 933ee51523a63e67430316e12b0c22119d51d1e20a659e6b909bfdd63e8e6852c38f56cf85f6737b7f436e4915570617db6bcba9dc3e44a88962b669410b2e8a
-  data.tar.gz: 9f9b7c82bcc74e005b03abe6e98b6e496821407a4adbc8a9314702cc090ac373b93ec7b03bd2195d88f17db8bbefe60c9c3edb1e997d18f66ab0eed0d3929df2
+  metadata.gz: d3ff0bb506bac3398d9f02e5c3023babc3a7534261de3e05cb0c06738c6cc586f1f94578c3fa9e4fba91468682d1cb7aca129fefff2fed23dea84543003862ab
+  data.tar.gz: d9acbeca7d5d942ec7f1454716e3b69750b25bdc83416253f86ca1c04e921774b58f968c0cedadee06ad82a2be11cb7951551d95523dd27f830e373f21b73490

data/fluent-plugin-pgjson.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-pgjson"
-  s.version     = "0.0.5"
+  s.version     = "0.0.6"
   s.authors     = ["OKUNO Akihiro"]
   s.email       = ["choplin.choplin@gmail.com"]
   s.homepage    = "https://github.com/choplin/fluent-plugin-pgjson"

data/lib/fluent/plugin/out_pgjson.rb

@@ -13,7 +13,7 @@ class PgJsonOutput < Fluent::BufferedOutput
   config_param :time_col   , :string  , :default => 'time'
   config_param :tag_col    , :string  , :default => 'tag'
   config_param :record_col , :string  , :default => 'record'
-  config_param :messagepack, :bool  , :default => false
+  config_param :msgpack    , :bool    , :default => false
 
   def initialize
     super
@@ -38,16 +38,18 @@ class PgJsonOutput < Fluent::BufferedOutput
   end
 
   def write(chunk)
+    init_connection
     begin
-      init_connection
       @conn.exec("COPY #{@table} (#{@tag_col}, #{@time_col}, #{@record_col}) FROM STDIN WITH DELIMITER E'\\x01'")
       chunk.msgpack_each do |tag, time, record|
         @conn.put_copy_data "#{tag}\x01#{Time.at(time).to_s}\x01#{record_value(record)}\n"
       end
       @conn.put_copy_end
     rescue
-      @conn.close()
-      @conn = nil
+      if ! @conn.nil?
+        @conn.close()
+        @conn = nil
+      end
       raise "failed to send data to postgres: #$!"
     end
   end
@@ -74,7 +76,9 @@ class PgJsonOutput < Fluent::BufferedOutput
     if @msgpack
       "\\#{@conn.escape_bytea(record.to_msgpack)}"
     else
-      record.to_json
+      json = record.to_json
+      json.gsub!(/\\/){ '\\\\' }
+      json
     end
   end
 end

data/test/plugin/test_out.rb

@@ -1,46 +1,133 @@
+require 'pg'
+require 'securerandom'
 require 'helper'
 
 class PgJsonOutputTest < Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-  end
+  HOST = "localhost"
+  PORT = 5432
+  DATABASE = "postgres"
+  TABLE = "test_fluentd_#{SecureRandom.hex}"
+  USER = "postgres"
+  PASSWORD = "postgres"
+
+  TIME_COL = "time"
+  TAG_COL = "tag"
+  RECORD_COL = "record"
 
   CONFIG = %[
     type pgjson
-    host localhost
-    port 5432
-    database fluentd
-    table fluentd
-    user postgres
-    password postgres
-    time_col time
-    tag_col tag
-    record_col record
+    host #{HOST}
+    port #{PORT}
+    database #{DATABASE}
+    table #{TABLE}
+    user #{USER}
+    password #{PASSWORD}
+    time_col #{TIME_COL}
+    tag_col #{TAG_COL}
+    record_col #{RECORD_COL}
   ]
 
-  def create_driver(conf = CONFIG, tag='test')
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::PgJsonOutput, tag).configure(conf)
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf = CONFIG, tag = 'test')
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::PgJsonOutput).configure(conf)
   end
 
   def test_configure
     d = create_driver
 
-    assert_equal "localhost", d.instance.host
-    assert_equal 5432, d.instance.port
-    assert_equal "fluentd", d.instance.database
-    assert_equal "fluentd", d.instance.table
-    assert_equal "postgres", d.instance.user
-    assert_equal "postgres", d.instance.password
-    assert_equal "time", d.instance.time_col
-    assert_equal "tag", d.instance.tag_col
-    assert_equal "record", d.instance.record_col
+    assert_equal HOST, d.instance.host
+    assert_equal PORT, d.instance.port
+    assert_equal DATABASE, d.instance.database
+    assert_equal TABLE, d.instance.table
+    assert_equal USER, d.instance.user
+    assert_equal PASSWORD, d.instance.password
+    assert_equal TIME_COL, d.instance.time_col
+    assert_equal TAG_COL, d.instance.tag_col
+    assert_equal RECORD_COL, d.instance.record_col
   end
 
-  def test_format
-    d = create_driver
+  def test_write
+    with_connection do |conn|
+      tag = 'test'
+      time = Time.parse("2014-12-26 07:58:37 UTC")
+      record = {"a"=>1}
+
+      d = create_driver(CONFIG, tag)
+      d.emit(record, time.to_i)
+      d.run
+      wait_for_data(conn)
+
+      res = conn.exec("select * from #{TABLE}")[0]
+      assert_equal res[TAG_COL], tag
+      assert_equal Time.parse(res[TIME_COL]), time
+      assert_equal res[RECORD_COL], record.to_json
+    end
   end
 
-  def test_write
-    d = create_driver
+  def test_escape_of_backslash
+    with_connection do |conn|
+      tag = 'test'
+      time = Time.parse("2014-12-26 07:58:37 UTC")
+      record = {"a"=>"\"foo\""}
+
+      d = create_driver(CONFIG, tag)
+      d.emit(record, time.to_i)
+      d.run
+      wait_for_data(conn)
+
+      res = conn.exec("select * from #{TABLE}")[0]
+      assert_equal res[TAG_COL], tag
+      assert_equal Time.parse(res[TIME_COL]), time
+      assert_equal res[RECORD_COL], record.to_json
+    end
+  end
+
+
+  def ensure_connection
+    conn = nil
+
+    assert_nothing_raised do
+      conn = PGconn.new(:dbname => DATABASE, :host => HOST, :port => PORT, :user => USER, :password => PASSWORD)
+    end
+
+    conn
+  end
+
+  def create_test_table(conn)
+    conn.exec(<<-SQL)
+      CREATE TABLE #{TABLE} (
+          #{TAG_COL} Text
+          ,#{TIME_COL} Timestamptz
+          ,#{RECORD_COL} Json
+      );
+    SQL
   end
+
+  def drop_test_table(conn)
+    conn.exec("DROP TABLE #{TABLE}")
+  end
+
+  def with_connection(&block)
+    conn = ensure_connection
+    create_test_table(conn)
+    begin
+      block.call(conn)
+    ensure
+      drop_test_table(conn)
+      conn.close
+    end
+  end
+
+  def wait_for_data(conn)
+    10.times do
+      res = conn.exec "select count(*) from #{TABLE}"
+      return if res.getvalue(0,0).to_i > 0
+      sleep 0.5
+    end
+    raise "COPY has not been finished correctly"
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-pgjson
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - OKUNO Akihiro
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-11 00:00:00.000000000 Z
+date: 2014-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd