fluent-plugin-pgjson 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 1677b7c020f8c8044319c6b2a42c587bc0fb4301
-  data.tar.gz: 4b5b242dc0cb85fc843ed338fe341fa795220b84
+SHA256:
+  metadata.gz: 7196d7c7b7181d2837c5a7e037caaf228335c1cac0b60b8212848d5b354652e3
+  data.tar.gz: a5c3f94dd289a27641c40daaa46a665b375afe9023ccbece49f44cd6338ff7fb
 SHA512:
-  metadata.gz: e5098cf14816bab706388fd80818c4269cb527da52ab451fc82a972506dc984dcca7697c49ece7bce483fd11762b6ec0a5b3c04b4698173daaadea0c7a7df687
-  data.tar.gz: bcf9756da26ae5120f90526b659606f1cada84dcd410be0b8bec69f44886a5cb6221068d7232d91aa0044551054dbc0475f7f8193107812140cecf68aff973af
+  metadata.gz: adc06d292c08d6bccc5b7ad4471abfbbe585af918044b306249f1c1f7ceb18eb3dd0eabbaedf2ddf04e62c88077d931288437be3faae7a272af731887d628e42
+  data.tar.gz: 7f627fdf20323d838f2254f7f64b101e5937e47b6113dd2e380759549195c23f26abf852fffc0679cd925bcfbb126736e486433343887e24f87eab4bd319752b

data/README.md

@@ -39,6 +39,12 @@ CREATE TABLE fluentd (
 );
 ```
 
+### Configurable JSON Encoder
+
+Fluentd's standard JSON encoder is `yajl`.
+`yajl` is robust for invalid byte sequence.
+But this plugin's default value is `json` which is Ruby standard JSON encoder for backward compatibility.
+
 ## Configuration
 
 ### Example
@@ -73,6 +79,8 @@ CREATE TABLE fluentd (
 |time_col|column name to insert time|time|
 |tag_col|column name to insert tag|tag|
 |record_col|column name to insert record|record|
+|msgpack|use msgpack format for inserting records|false|
+|encoder|choose prefer JSON encoder (yajl/json)|json|
 
 ## Copyright
 

data/fluent-plugin-pgjson.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-pgjson"
-  s.version     = "0.0.8"
+  s.version     = "0.0.9"
   s.authors     = ["OKUNO Akihiro"]
   s.email       = ["choplin.choplin@gmail.com"]
   s.homepage    = "https://github.com/choplin/fluent-plugin-pgjson"
@@ -18,4 +18,6 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency "fluentd"
   s.add_runtime_dependency "pg"
+  s.add_development_dependency "test-unit", ">= 3.1.0"
+  s.add_development_dependency "rake", ">= 11.0"
 end

data/lib/fluent/plugin/out_pgjson.rb

@@ -1,8 +1,17 @@
-module Fluent
+require 'fluent/plugin/output'
+require 'pg'
+require 'yajl'
+require 'json'
 
-class PgJsonOutput < Fluent::BufferedOutput
+module Fluent::Plugin
+
+class PgJsonOutput < Fluent::Output
   Fluent::Plugin.register_output('pgjson', self)
 
+  helpers :compat_parameters
+
+  DEFAULT_BUFFER_TYPE = "memory"
+
   config_param :host       , :string  , :default => 'localhost'
   config_param :port       , :integer , :default => 5432
   config_param :sslmode    , :string  , :default => 'prefer'
@@ -14,35 +23,60 @@ class PgJsonOutput < Fluent::BufferedOutput
   config_param :tag_col    , :string  , :default => 'tag'
   config_param :record_col , :string  , :default => 'record'
   config_param :msgpack    , :bool    , :default => false
+  config_param :encoder    , :enum, list: [:yajl, :json], :default => :json
+  config_param :time_format, :string  , :default => '%F %T.%N %z'
+
+  config_section :buffer do
+    config_set_default :@type, DEFAULT_BUFFER_TYPE
+    config_set_default :chunk_keys, ['tag']
+  end
 
   def initialize
     super
-    require 'pg'
     @conn = nil
   end
 
   def configure(conf)
+    compat_parameters_convert(conf, :buffer)
     super
+    unless @chunk_key_tag
+      raise Fluent::ConfigError, "'tag' in chunk_keys is required."
+    end
+    @encoder = case @encoder
+               when :yajl
+                 Yajl
+               when :json
+                 JSON
+               end
   end
 
   def shutdown
-    super
-
     if ! @conn.nil? and ! @conn.finished?
       @conn.close()
     end
+
+    super
+  end
+
+  def formatted_to_msgpack_binary
+    true
+  end
+
+  def multi_workers_ready?
+    true
   end
 
   def format(tag, time, record)
-    [tag, time, record].to_msgpack
+    [Time.at(time).strftime(@time_format), record].to_msgpack
   end
 
   def write(chunk)
     init_connection
     @conn.exec("COPY #{@table} (#{@tag_col}, #{@time_col}, #{@record_col}) FROM STDIN WITH DELIMITER E'\\x01'")
     begin
-      chunk.msgpack_each do |tag, time, record|
-        @conn.put_copy_data "#{tag}\x01#{Time.at(time).to_s}\x01#{record_value(record)}\n"
+      tag = chunk.metadata.tag
+      chunk.msgpack_each do |time, record|
+        @conn.put_copy_data "#{tag}\x01#{time}\x01#{record_value(record)}\n"
       end
     rescue => err
       errmsg = "%s while copy data: %s" % [ err.class.name, err.message ]
@@ -62,7 +96,7 @@ class PgJsonOutput < Fluent::BufferedOutput
       $log.debug "connecting to PostgreSQL server #{@host}:#{@port}, database #{@database}..."
 
       begin
-        @conn = PGconn.new(:dbname => @database, :host => @host, :port => @port, :sslmode => @sslmode, :user => @user, :password => @password)
+        @conn = PG::Connection.new(:dbname => @database, :host => @host, :port => @port, :sslmode => @sslmode, :user => @user, :password => @password)
       rescue
         if ! @conn.nil?
           @conn.close()
@@ -77,7 +111,7 @@ class PgJsonOutput < Fluent::BufferedOutput
     if @msgpack
       "\\#{@conn.escape_bytea(record.to_msgpack)}"
     else
-      json = record.to_json
+      json = @encoder.dump(record)
       json.gsub!(/\\/){ '\\\\' }
       json
     end

data/test/helper.rb

@@ -1,27 +1,7 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
+require 'bundler/setup'
 require 'test/unit'
 
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
+$LOAD_PATH.unshift(__dir__)
 require 'fluent/test'
-unless ENV.has_key?('VERBOSE')
-  nulllogger = Object.new
-  nulllogger.instance_eval {|obj|
-    def method_missing(method, *args)
-      # pass
-    end
-  }
-  $log = nulllogger
-end
 require 'fluent/plugin/out_pgjson'
-
-class Test::Unit::TestCase
-end

data/test/plugin/test_out.rb

@@ -1,18 +1,23 @@
 require 'pg'
 require 'securerandom'
 require 'helper'
+require 'fluent/test/driver/output'
+require 'fluent/test/helpers'
 
 class PgJsonOutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
   HOST = "localhost"
   PORT = 5432
   DATABASE = "postgres"
   TABLE = "test_fluentd_#{SecureRandom.hex}"
-  USER = "postgres"
-  PASSWORD = "postgres"
+  USER = ENV["PSQL_USER"] || "postgres"
+  PASSWORD = ENV["PSQL_PASSWORD"] || "postgres"
 
   TIME_COL = "time"
   TAG_COL = "tag"
   RECORD_COL = "record"
+  ENCODER = JSON
 
   CONFIG = %[
     type pgjson
@@ -31,8 +36,8 @@ class PgJsonOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
-  def create_driver(conf = CONFIG, tag = 'test')
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::PgJsonOutput).configure(conf)
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::PgJsonOutput).configure(conf)
   end
 
   def test_configure
@@ -47,22 +52,46 @@ class PgJsonOutputTest < Test::Unit::TestCase
     assert_equal TIME_COL, d.instance.time_col
     assert_equal TAG_COL, d.instance.tag_col
     assert_equal RECORD_COL, d.instance.record_col
+    assert_equal ENCODER, d.instance.encoder
+  end
+
+  def test_invalid_chunk_keys
+    assert_raise_message(/'tag' in chunk_keys is required./) do
+      create_driver(Fluent::Config::Element.new(
+                      'ROOT', '', {
+                        '@type' => 'pgjson',
+                        'host' => "#{HOST}",
+                        'port' => "#{PORT}",
+                        'database' => "#{DATABASE}",
+                        'table' => "#{TABLE}",
+                        'user' => "#{USER}",
+                        'password' => "#{PASSWORD}",
+                        'time_col' => "#{TIME_COL}",
+                        'tag_col' => "#{TAG_COL}",
+                        'record_col' => "#{RECORD_COL}",
+                      }, [
+                        Fluent::Config::Element.new('buffer', 'mykey', {
+                                                      'chunk_keys' => 'mykey'
+                                                    }, [])
+                      ]))
+    end
   end
 
   def test_write
     with_connection do |conn|
       tag = 'test'
-      time = Time.parse("2014-12-26 07:58:37 UTC")
+      time = event_time("2014-12-26 07:58:37 UTC")
       record = {"a"=>1}
 
-      d = create_driver(CONFIG, tag)
-      d.emit(record, time.to_i)
-      d.run
+      d = create_driver(CONFIG)
+      d.run(default_tag: tag) do
+        d.feed(time, record)
+      end
       wait_for_data(conn)
 
       res = conn.exec("select * from #{TABLE}")[0]
       assert_equal res[TAG_COL], tag
-      assert_equal Time.parse(res[TIME_COL]), time
+      assert_equal event_time(res[TIME_COL]), time
       assert_equal res[RECORD_COL], record.to_json
     end
   end
@@ -70,17 +99,18 @@ class PgJsonOutputTest < Test::Unit::TestCase
   def test_escape_of_backslash
     with_connection do |conn|
       tag = 'test'
-      time = Time.parse("2014-12-26 07:58:37 UTC")
+      time = event_time("2014-12-26 07:58:37 UTC")
       record = {"a"=>"\"foo\""}
 
-      d = create_driver(CONFIG, tag)
-      d.emit(record, time.to_i)
-      d.run
+      d = create_driver(CONFIG)
+      d.run(default_tag: tag) do
+        d.feed(time, record)
+      end
       wait_for_data(conn)
 
       res = conn.exec("select * from #{TABLE}")[0]
       assert_equal res[TAG_COL], tag
-      assert_equal Time.parse(res[TIME_COL]), time
+      assert_equal event_time(res[TIME_COL]), time
       assert_equal res[RECORD_COL], record.to_json
     end
   end
@@ -88,17 +118,18 @@ class PgJsonOutputTest < Test::Unit::TestCase
   def test_invalid_json
     with_connection do |conn|
       tag = 'test'
-      time = Time.parse("2014-12-26 07:58:37 UTC")
+      time = event_time("2014-12-26 07:58:37 UTC")
 
-      d = create_driver(CONFIG, tag)
+      d = create_driver(CONFIG)
       instance = d.instance
       def instance.record_value(record)
         'invalid json'
       end
-      d.emit('', time.to_i)
 
       assert_raise RuntimeError do
-        d.run
+        d.run(default_tag: tag) do
+          d.feed(time, {})
+        end
       end
     end
   end
@@ -108,7 +139,7 @@ class PgJsonOutputTest < Test::Unit::TestCase
     conn = nil
 
     assert_nothing_raised do
-      conn = PGconn.new(:dbname => DATABASE, :host => HOST, :port => PORT, :user => USER, :password => PASSWORD)
+      conn = PG::Connection.new(:dbname => DATABASE, :host => HOST, :port => PORT, :user => USER, :password => PASSWORD)
     end
 
     conn

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-pgjson
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - OKUNO Akihiro
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-01 00:00:00.000000000 Z
+date: 2018-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -38,6 +38,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '11.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '11.0'
 description: ''
 email:
 - choplin.choplin@gmail.com
@@ -75,7 +103,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: ''