fluent-plugin-pgjson 0.0.9 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7196d7c7b7181d2837c5a7e037caaf228335c1cac0b60b8212848d5b354652e3
-  data.tar.gz: a5c3f94dd289a27641c40daaa46a665b375afe9023ccbece49f44cd6338ff7fb
+  metadata.gz: a4583a2849d4add68b6735fd279bed9d975fc9f78e88043cf07dee81bbcbb4f4
+  data.tar.gz: 2b13f3c83f4b35f605450c304b01d1a2510f5928609847aa7036f2ad885ec362
 SHA512:
-  metadata.gz: adc06d292c08d6bccc5b7ad4471abfbbe585af918044b306249f1c1f7ceb18eb3dd0eabbaedf2ddf04e62c88077d931288437be3faae7a272af731887d628e42
-  data.tar.gz: 7f627fdf20323d838f2254f7f64b101e5937e47b6113dd2e380759549195c23f26abf852fffc0679cd925bcfbb126736e486433343887e24f87eab4bd319752b
+  metadata.gz: ee23b29a5a9d27a29be7bfa42e1ba4970e7876a2c422d535b9f47fb155a45c606646b5857a3865d4044bc7639a1966f4af6757f407d55ae47dac9026300f3c92
+  data.tar.gz: 7cb84bc39f1bfa14be686a31a6b14d3f47bf552c368b99ed4dbed8844ee6b726e514f5957d01a60009ecb03a9883871fb0d34c004f3cb3988a3e1bb3d6bec1e9

data/.travis.yml

@@ -0,0 +1,17 @@
+language: ruby
+
+rvm:
+  - 2.2
+  - 2.3
+  - 2.4
+  - 2.5
+
+addons:
+  postgresql: "9.6"
+
+services:
+  - postgresql
+
+before_script:
+  - psql -c "create role fluentd with LOGIN CREATEDB password 'fluentd';" -U postgres
+  - psql -c "create database pgjsontest owner fluentd encoding 'utf-8'" -U postgres

data/Gemfile

@@ -1,4 +1,4 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
 # Specify your gem's dependencies in fluent-plugin-pgjson.gemspec
 gemspec

data/README.md

@@ -4,19 +4,28 @@ Output Plugin for PostgreSQL Json Type.
 
 <b>Json type is availble in PostgreSQL version over 9.2</b>
 
+## Requirements
+
+| fluent-plugin-pgjson | fluentd    | Ruby   |
+|----------------------|------------|--------|
+| >= 1.0.0             | >= v1.0.0  | >= 2.2 |
+|  < 1.0.0             | >= v0.12.0 | >= 1.9 |
+
 ## Installation
 
-`$ fluent-gem install fluent-plugin-pgjson`
+```
+$ fluent-gem install fluent-plugin-pgjson
+```
 
 ## Schema
 
-Specified table must have this schema.
+Specified table must have following schema:
 
-|col|type|
-|---|---|
-|{tag_col}|Text|
-|{time_col}|Timestamp WITH TIME ZONE|
-|{record_col}|Json|
+| col          | type                     |
+|--------------|--------------------------|
+| {tag_col}    | Text                     |
+| {time_col}   | Timestamp WITH TIME ZONE |
+| {record_col} | Json                     |
 
 ### Example
 
@@ -39,19 +48,13 @@ CREATE TABLE fluentd (
 );
 ```
 
-### Configurable JSON Encoder
-
-Fluentd's standard JSON encoder is `yajl`.
-`yajl` is robust for invalid byte sequence.
-But this plugin's default value is `json` which is Ruby standard JSON encoder for backward compatibility.
-
 ## Configuration
 
 ### Example
 
 ```
 <match **>
-  type pgjson
+  @type pgjson
   host localhost
   port 5432
   sslmode require
@@ -67,20 +70,27 @@ But this plugin's default value is `json` which is Ruby standard JSON encoder fo
 
 ### Parameter
 
-|parameter|description|default|
-|---|---|---|
-|host|postgres server hostname|localhost|
-|port|postgres server port number|5432|
-|sslmode|use ssl (disable/allow/prefer/require)|prefer||
-|database|database name to which records will be inserted||
-|table|table name to which records will be inserted||
-|user|user name used to connect database|nil|
-|password|password uset to connect database|nil|
-|time_col|column name to insert time|time|
-|tag_col|column name to insert tag|tag|
-|record_col|column name to insert record|record|
-|msgpack|use msgpack format for inserting records|false|
-|encoder|choose prefer JSON encoder (yajl/json)|json|
+* **host** (string) (optional): The hostname of PostgreSQL server
+  * Default value: `localhost`.
+* **port** (integer) (optional): The port of PostgreSQL server
+  * Default value: `5432`.
+* **sslmode** (enum) (optional): Set the sslmode to enable Eavesdropping protection/MITM protection. See [PostgreSQL Documentation](https://www.postgresql.org/docs/10/static/libpq-ssl.html) for more details.
+  * Available values: disable, allow, prefer, require, verify-ca, verify-full
+  * Default value: `prefer`.
+* **database** (string) (required): The database name to connect
+* **table** (string) (required): The table name to insert records
+* **user** (string) (optional): The user name to connect database
+* **password** (string) (optional): The password to connect database
+* **time_col** (string) (optional): The column name for the time
+  * Default value: `time`.
+* **tag_col** (string) (optional): The column name for the tag
+  * Default value: `tag`.
+* **record_col** (string) (optional): The column name for the record
+  * Default value: `record`.
+* **msgpack** (bool) (optional): If true, insert records formatted as msgpack
+* **encoder** (enum) (optional): JSON encoder (yajl/json)
+  * Available values: yajl, json
+  * Default value: `yajl`.
 
 ## Copyright
 

data/Rakefile

@@ -1,10 +1,10 @@
 #!/usr/bin/env rake
 require "bundler/gem_tasks"
 
-require 'rake/testtask'
+require "rake/testtask"
 Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
+  test.libs << "lib" << "test"
+  test.pattern = "test/**/test_*.rb"
   test.verbose = true
 end
 

data/fluent-plugin-pgjson.gemspec

@@ -1,13 +1,12 @@
-# -*- encoding: utf-8 -*-
 $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-pgjson"
-  s.version     = "0.0.9"
+  s.version     = "1.0.0"
   s.authors     = ["OKUNO Akihiro"]
   s.email       = ["choplin.choplin@gmail.com"]
-  s.homepage    = "https://github.com/choplin/fluent-plugin-pgjson"
-  s.summary     = %q{}
+  s.homepage    = "https://github.com/fluent-plugins-nursery/fluent-plugin-pgjson"
+  s.summary     = %q{Fluentd Output Plugin for PostgreSQL JSON Type.}
   s.description = %q{}
   s.license  = "Apache-2.0"
 
@@ -16,7 +15,7 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_runtime_dependency "fluentd"
+  s.add_runtime_dependency "fluentd", ">=1.0.0", "<2"
   s.add_runtime_dependency "pg"
   s.add_development_dependency "test-unit", ">= 3.1.0"
   s.add_development_dependency "rake", ">= 11.0"

data/lib/fluent/plugin/out_pgjson.rb

@@ -1,121 +1,133 @@
-require 'fluent/plugin/output'
-require 'pg'
-require 'yajl'
-require 'json'
+require "fluent/plugin/output"
+require "pg"
+require "yajl"
+require "json"
 
 module Fluent::Plugin
+  class PgJsonOutput < Fluent::Plugin::Output
+    Fluent::Plugin.register_output("pgjson", self)
+
+    helpers :compat_parameters
+
+    DEFAULT_BUFFER_TYPE = "memory"
+
+    desc "The hostname of PostgreSQL server"
+    config_param :host, :string, default: "localhost"
+    desc "The port of PostgreSQL server"
+    config_param :port, :integer, default: 5432
+    desc "Set the sslmode to enable Eavesdropping protection/MITM protection"
+    config_param :sslmode, :enum, list: %i[disable allow prefer require verify-ca verify-full], default: :prefer
+    desc "The database name to connect"
+    config_param :database, :string
+    desc "The table name to insert records"
+    config_param :table, :string
+    desc "The user name to connect database"
+    config_param :user, :string, default: nil
+    desc "The password to connect database"
+    config_param :password, :string, default: nil, secret: true
+    desc "The column name for the time"
+    config_param :time_col, :string, default: "time"
+    desc "The column name for the tag"
+    config_param :tag_col, :string, default: "tag"
+    desc "The column name for the record"
+    config_param :record_col, :string, default: "record"
+    desc "If true, insert records formatted as msgpack"
+    config_param :msgpack, :bool, default: false
+    desc "JSON encoder (yajl/json)"
+    config_param :encoder, :enum, list: [:yajl, :json], default: :yajl
+
+    config_param :time_format, :string, default: "%F %T.%N %z"
+
+    config_section :buffer do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+      config_set_default :chunk_keys, ["tag"]
+    end
 
-class PgJsonOutput < Fluent::Output
-  Fluent::Plugin.register_output('pgjson', self)
-
-  helpers :compat_parameters
-
-  DEFAULT_BUFFER_TYPE = "memory"
-
-  config_param :host       , :string  , :default => 'localhost'
-  config_param :port       , :integer , :default => 5432
-  config_param :sslmode    , :string  , :default => 'prefer'
-  config_param :database   , :string
-  config_param :table      , :string
-  config_param :user       , :string  , :default => nil
-  config_param :password   , :string  , :default => nil , :secret => true
-  config_param :time_col   , :string  , :default => 'time'
-  config_param :tag_col    , :string  , :default => 'tag'
-  config_param :record_col , :string  , :default => 'record'
-  config_param :msgpack    , :bool    , :default => false
-  config_param :encoder    , :enum, list: [:yajl, :json], :default => :json
-  config_param :time_format, :string  , :default => '%F %T.%N %z'
-
-  config_section :buffer do
-    config_set_default :@type, DEFAULT_BUFFER_TYPE
-    config_set_default :chunk_keys, ['tag']
-  end
-
-  def initialize
-    super
-    @conn = nil
-  end
-
-  def configure(conf)
-    compat_parameters_convert(conf, :buffer)
-    super
-    unless @chunk_key_tag
-      raise Fluent::ConfigError, "'tag' in chunk_keys is required."
+    def initialize
+      super
+      @conn = nil
     end
-    @encoder = case @encoder
-               when :yajl
-                 Yajl
-               when :json
-                 JSON
-               end
-  end
 
-  def shutdown
-    if ! @conn.nil? and ! @conn.finished?
-      @conn.close()
+    def configure(conf)
+      compat_parameters_convert(conf, :buffer)
+      super
+      unless @chunk_key_tag
+        raise Fluent::ConfigError, "'tag' in chunk_keys is required."
+      end
+      @encoder = case @encoder
+                 when :yajl
+                   Yajl
+                 when :json
+                   JSON
+                 end
     end
 
-    super
-  end
+    def shutdown
+      if ! @conn.nil? and ! @conn.finished?
+        @conn.close()
+      end
 
-  def formatted_to_msgpack_binary
-    true
-  end
+      super
+    end
 
-  def multi_workers_ready?
-    true
-  end
+    def formatted_to_msgpack_binary
+      true
+    end
 
-  def format(tag, time, record)
-    [Time.at(time).strftime(@time_format), record].to_msgpack
-  end
+    def multi_workers_ready?
+      true
+    end
+
+    def format(tag, time, record)
+      [Time.at(time).strftime(@time_format), record].to_msgpack
+    end
 
-  def write(chunk)
-    init_connection
-    @conn.exec("COPY #{@table} (#{@tag_col}, #{@time_col}, #{@record_col}) FROM STDIN WITH DELIMITER E'\\x01'")
-    begin
-      tag = chunk.metadata.tag
-      chunk.msgpack_each do |time, record|
-        @conn.put_copy_data "#{tag}\x01#{time}\x01#{record_value(record)}\n"
+    def write(chunk)
+      init_connection
+      @conn.exec("COPY #{@table} (#{@tag_col}, #{@time_col}, #{@record_col}) FROM STDIN WITH DELIMITER E'\\x01'")
+      begin
+        tag = chunk.metadata.tag
+        chunk.msgpack_each do |time, record|
+          @conn.put_copy_data "#{tag}\x01#{time}\x01#{record_value(record)}\n"
+        end
+      rescue => err
+        errmsg = "%s while copy data: %s" % [ err.class.name, err.message ]
+        @conn.put_copy_end( errmsg )
+        @conn.get_result
+        raise
+      else
+        @conn.put_copy_end
+        res = @conn.get_result
+        raise res.result_error_message if res.result_status!=PG::PGRES_COMMAND_OK
       end
-    rescue => err
-      errmsg = "%s while copy data: %s" % [ err.class.name, err.message ]
-      @conn.put_copy_end( errmsg )
-      @conn.get_result
-      raise
-    else
-      @conn.put_copy_end
-      res = @conn.get_result
-      raise res.result_error_message if res.result_status!=PG::PGRES_COMMAND_OK
     end
-  end
 
-  private
-  def init_connection
-    if @conn.nil?
-      $log.debug "connecting to PostgreSQL server #{@host}:#{@port}, database #{@database}..."
+    private
 
-      begin
-        @conn = PG::Connection.new(:dbname => @database, :host => @host, :port => @port, :sslmode => @sslmode, :user => @user, :password => @password)
-      rescue
-        if ! @conn.nil?
-          @conn.close()
-          @conn = nil
+    def init_connection
+      if @conn.nil?
+        log.debug "connecting to PostgreSQL server #{@host}:#{@port}, database #{@database}..."
+
+        begin
+          @conn = PG::Connection.new(dbname: @database, host: @host, port: @port, sslmode: @sslmode, user: @user, password: @password)
+        rescue
+          if ! @conn.nil?
+            @conn.close()
+            @conn = nil
+          end
+          raise "failed to initialize connection: #$!"
         end
-        raise "failed to initialize connection: #$!"
       end
     end
-  end
 
-  def record_value(record)
-    if @msgpack
-      "\\#{@conn.escape_bytea(record.to_msgpack)}"
-    else
-      json = @encoder.dump(record)
-      json.gsub!(/\\/){ '\\\\' }
-      json
+    def record_value(record)
+      if @msgpack
+        "\\#{@conn.escape_bytea(record.to_msgpack)}"
+      else
+        json = @encoder.dump(record)
+        json.gsub!(/\\/){ '\\\\' }
+        json
+      end
     end
   end
 end
-
-end

data/test/helper.rb

@@ -1,7 +1,7 @@
-require 'bundler/setup'
-require 'test/unit'
+require "bundler/setup"
+require "test/unit"
 
-$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
+$LOAD_PATH.unshift(File.join(__dir__, "..", "lib"))
 $LOAD_PATH.unshift(__dir__)
-require 'fluent/test'
-require 'fluent/plugin/out_pgjson'
+require "fluent/test"
+require "fluent/plugin/out_pgjson"

data/test/plugin/test_out.rb

@@ -1,26 +1,26 @@
-require 'pg'
-require 'securerandom'
-require 'helper'
-require 'fluent/test/driver/output'
-require 'fluent/test/helpers'
+require "pg"
+require "securerandom"
+require "helper"
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
 
 class PgJsonOutputTest < Test::Unit::TestCase
   include Fluent::Test::Helpers
 
   HOST = "localhost"
   PORT = 5432
-  DATABASE = "postgres"
+  DATABASE = "pgjsontest"
   TABLE = "test_fluentd_#{SecureRandom.hex}"
-  USER = ENV["PSQL_USER"] || "postgres"
-  PASSWORD = ENV["PSQL_PASSWORD"] || "postgres"
+  USER = "fluentd"
+  PASSWORD = "fluentd"
 
   TIME_COL = "time"
   TAG_COL = "tag"
   RECORD_COL = "record"
-  ENCODER = JSON
+  ENCODER = Yajl
 
   CONFIG = %[
-    type pgjson
+    @type pgjson
     host #{HOST}
     port #{PORT}
     database #{DATABASE}
@@ -58,20 +58,20 @@ class PgJsonOutputTest < Test::Unit::TestCase
   def test_invalid_chunk_keys
     assert_raise_message(/'tag' in chunk_keys is required./) do
       create_driver(Fluent::Config::Element.new(
-                      'ROOT', '', {
-                        '@type' => 'pgjson',
-                        'host' => "#{HOST}",
-                        'port' => "#{PORT}",
-                        'database' => "#{DATABASE}",
-                        'table' => "#{TABLE}",
-                        'user' => "#{USER}",
-                        'password' => "#{PASSWORD}",
-                        'time_col' => "#{TIME_COL}",
-                        'tag_col' => "#{TAG_COL}",
-                        'record_col' => "#{RECORD_COL}",
+                      "ROOT", "", {
+                        "@type" => "pgjson",
+                        "host" => "#{HOST}",
+                        "port" => "#{PORT}",
+                        "database" => "#{DATABASE}",
+                        "table" => "#{TABLE}",
+                        "user" => "#{USER}",
+                        "password" => "#{PASSWORD}",
+                        "time_col" => "#{TIME_COL}",
+                        "tag_col" => "#{TAG_COL}",
+                        "record_col" => "#{RECORD_COL}",
                       }, [
-                        Fluent::Config::Element.new('buffer', 'mykey', {
-                                                      'chunk_keys' => 'mykey'
+                        Fluent::Config::Element.new("buffer", "mykey", {
+                                                      "chunk_keys" => "mykey"
                                                     }, [])
                       ]))
     end
@@ -79,7 +79,7 @@ class PgJsonOutputTest < Test::Unit::TestCase
 
   def test_write
     with_connection do |conn|
-      tag = 'test'
+      tag = "test"
       time = event_time("2014-12-26 07:58:37 UTC")
       record = {"a"=>1}
 
@@ -98,7 +98,7 @@ class PgJsonOutputTest < Test::Unit::TestCase
 
   def test_escape_of_backslash
     with_connection do |conn|
-      tag = 'test'
+      tag = "test"
       time = event_time("2014-12-26 07:58:37 UTC")
       record = {"a"=>"\"foo\""}
 
@@ -117,13 +117,13 @@ class PgJsonOutputTest < Test::Unit::TestCase
 
   def test_invalid_json
     with_connection do |conn|
-      tag = 'test'
+      tag = "test"
       time = event_time("2014-12-26 07:58:37 UTC")
 
       d = create_driver(CONFIG)
       instance = d.instance
       def instance.record_value(record)
-        'invalid json'
+        "invalid json"
       end
 
       assert_raise RuntimeError do
@@ -139,7 +139,7 @@ class PgJsonOutputTest < Test::Unit::TestCase
     conn = nil
 
     assert_nothing_raised do
-      conn = PG::Connection.new(:dbname => DATABASE, :host => HOST, :port => PORT, :user => USER, :password => PASSWORD)
+      conn = PG::Connection.new(dbname: DATABASE, host: HOST, port: PORT, user: USER, password: PASSWORD)
     end
 
     conn

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-pgjson
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 1.0.0
 platform: ruby
 authors:
 - OKUNO Akihiro
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-24 00:00:00.000000000 Z
+date: 2018-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,14 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -74,6 +80,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE
 - README.md
@@ -83,7 +90,7 @@ files:
 - lib/fluent/plugin/out_pgjson.rb
 - test/helper.rb
 - test/plugin/test_out.rb
-homepage: https://github.com/choplin/fluent-plugin-pgjson
+homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-pgjson
 licenses:
 - Apache-2.0
 metadata: {}
@@ -106,7 +113,5 @@ rubyforge_project:
 rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
-summary: ''
-test_files:
-- test/helper.rb
-- test/plugin/test_out.rb
+summary: Fluentd Output Plugin for PostgreSQL JSON Type.
+test_files: []