fluent-plugin-postgres-bulk 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e9beccd75e6813dc745d12c56a8f097aad875bbca6811ec6f93bd8d8246d0538
+  data.tar.gz: 7fc02ae6abea7c575d9acbc9482f84e5f2d45470ce84a46386cd0480568d0283
+SHA512:
+  metadata.gz: '0716585370c79fe0e3fd09e2448d7e12814ebc7d2043354a5786c5e209efa01045fa974a159abd0148ddb0389607152d5c4c67da1fdac70c78dcf64b378ede15'
+  data.tar.gz: 7d3581fb539b80944d4e4d82b7f6f36d01f3b161b49c2375d97383a7aef176e3300e0de2b01711487f23c6e49b3923aa1f2e5f003e294dc57cedef05b753a798

data/.gitignore

@@ -0,0 +1,3 @@
+Gemfile.lock
+test/plugin/docker/plugins/
+pkg

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,102 @@
+fluent-plugin-postgres-bulk
+=====
+
+[Fluentd](https://www.fluentd.org/) output plugin to postgres.
+
+# Usage
+
+```
+<match **>
+  @type postgres_bulk
+  host my-host-name.local
+  port 5432
+  username ksss
+  password password
+  table any
+  column_names id,col1,col2,col3
+</match>
+```
+
+Plugin build and exec query to postgres like this
+
+```sql
+INSERT INTO #{@table} (#{column_names}) VALUES (...),(...),(...);
+```
+
+# Configuration
+
+* See also: [Output Plugin Overview](https://docs.fluentd.org/v1.0/articles/output-plugin-overview)
+
+## Fluent::Plugin::PostgresBulkOutput
+
+### host (string) (optional)
+
+Database host.
+
+Default value: `127.0.0.1`.
+
+### port (integer) (optional)
+
+Database port.
+
+Default value: `5432`.
+
+### database (string) (required)
+
+Database name.
+
+### username (string) (required)
+
+Database user.
+
+### password (string) (optional) (secret)
+
+Database password.
+
+Default value: `(empty)`.
+
+### table (string) (required)
+
+Bulk insert table.
+
+### column_names (array) (required)
+
+Bulk insert column.
+
+# Installation
+
+## RubyGems
+
+```
+$ gem install fluent-plugin-postgres-bulk
+```
+
+## Bundler
+
+Add following line to your Gemfile:
+
+```ruby
+gem "fluent-plugin-postgres-bulk"
+```
+
+And then execute:
+
+```
+$ bundle
+```
+
+# Require
+
+- libpq(build pg gem)
+
+# Contributing
+
+I need your help.
+
+# Copyright
+
+Copyright (c) 2018 Yuki Kurihara.
+
+# LISENCE
+
+MIT

data/Rakefile

@@ -0,0 +1,15 @@
+require "bundler/gem_tasks"
+require 'fileutils'
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+
+  if ENV["WITH_DOCKER"]
+    file "test/plugin/docker/plugins/out_postgres_bulk.rb" => "lib/fluent/plugin/out_postgres_bulk.rb"
+    FileUtils.cp "lib/fluent/plugin/out_postgres_bulk.rb", "test/plugin/docker/plugins/out_postgres_bulk.rb"
+  end
+end
+
+task :default => :test

data/fluent-plugin-postgres-bulk.gemspec

@@ -0,0 +1,21 @@
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-postgres-bulk"
+  gem.version       = "0.1.0"
+  gem.authors       = ["ksss"]
+  gem.email         = ["co000ri@gmail.com"]
+  gem.description   = %q{fluent plugin for bulk insert to postgres}
+  gem.summary       = %q{fluent plugin for bulk insert to postgres}
+  gem.homepage      = "https://github.com/ksss/fluent-plugin-postgres-bulk"
+  gem.license       = "MIT"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency "fluentd", ['>= 0.14.8', '< 2']
+  gem.add_runtime_dependency "pg"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit"
+  gem.add_development_dependency "fluent-logger"
+end

data/lib/fluent/plugin/out_postgres_bulk.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+require 'fluent/plugin/output'
+require 'benchmark'
+
+module Fluent::Plugin
+  class PostgresBulkOutput < Output
+    # number of parameters must be between 0 and 65535
+    N_PARAMETER_MAX = 65535
+    Fluent::Plugin.register_output('postgres_bulk', self)
+
+    config_param :host, :string, default: '127.0.0.1',
+      desc: "Database host."
+    config_param :port, :integer, default: 5432,
+      desc: "Database port."
+    config_param :database, :string,
+      desc: "Database name."
+    config_param :username, :string,
+      desc: "Database user."
+    config_param :password, :string, default: '', secret: true,
+      desc: "Database password."
+    config_param :table, :string,
+      desc: "Bulk insert table."
+    config_param :column_names, :array,
+      desc: "Bulk insert column."
+
+    def initialize
+      super
+      require 'pg'
+    end
+
+    def client
+      PG.connect(
+        host: @host,
+        port: @port,
+        dbname: @database,
+        user: @username,
+        password: @password,
+      )
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def write(chunk)
+      handler = client()
+      values = build_values(chunk)
+      max_slice = N_PARAMETER_MAX / @column_names.length * @column_names.length
+      values.each_slice(max_slice) do |slice|
+        place_holders = build_place_holders(slice)
+        query = "INSERT INTO #{@table} (#{@column_names.join(',')}) VALUES #{place_holders}"
+        t = Benchmark.realtime {
+          handler.exec_params(query, slice)
+        }
+        @log.debug("inserted #{values.length / @column_names.length} records in #{(t * 1000).to_i} ms")
+      end
+    ensure
+      handler.close
+    end
+
+    private
+
+    def build_values(chunk)
+      values = []
+      chunk.each { |time, record|
+        v = @column_names.map { |k|
+          record[k]
+        }
+        values.push(*v)
+      }
+      values
+    end
+
+    def build_place_holders(values)
+      values.each_slice(@column_names.length)
+            .map
+            .with_index { |cols, i|
+              params = cols.map.with_index { |c, j|
+                "$#{i * cols.length + j + 1}"
+              }
+              "(#{params.join(',')})"
+            }.join(',')
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/docker/Dockerfile-fluentd

@@ -0,0 +1,11 @@
+FROM fluent/fluentd:v1.2
+
+RUN apk add --update --virtual .build-deps sudo build-base ruby-dev \
+    ruby-bundler postgresql-dev
+COPY Gemfile* /fluentd/etc/
+RUN cd /fluentd/etc \
+  && bundle install -j4 \
+    --deployment \
+    --path /fluent/etc/vender/bundle
+COPY fluent.conf /fluentd/etc/fluent.conf
+COPY plugins /fluentd/plugins/

data/test/plugin/docker/Dockerfile-postgres

@@ -0,0 +1,3 @@
+FROM postgres:9.6
+
+COPY init.sql /docker-entrypoint-initdb.d/

data/test/plugin/docker/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+gem "fluentd", "1.2.0"
+gem "fluent-plugin-record-modifier"
+gem "json"
+gem "pg"

data/test/plugin/docker/docker-compose.yml

@@ -0,0 +1,23 @@
+version: "3"
+services:
+
+  postgres:
+    build:
+      context: .
+      dockerfile: Dockerfile-postgres
+    restart: always
+    environment:
+      POSTGRES_DB: test
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: password
+    ports:
+      - '5432:5432'
+
+  fluentd:
+    build:
+      context: .
+      dockerfile: Dockerfile-fluentd
+    environment:
+      FLUENTD_OPT: -v --gemfile /fluentd/etc/Gemfile
+    ports:
+      - '24224:24224'

data/test/plugin/docker/fluent.conf

@@ -0,0 +1,33 @@
+<source>
+  @type  forward
+  @id    input1
+  @label @mainstream
+  port  24224
+</source>
+
+<label @mainstream>
+  <match **>
+    @type copy
+    <store>
+      @type postgres_bulk
+      @id output1
+      host postgres
+      port 5432
+      username postgres
+      password password
+      database test
+      table bulk
+      column_names col4,col2,col3,col1
+      <buffer tag>
+        @type file
+        path /fluentd/log/bulk
+        flush_thread_count 2
+        flush_at_shutdown true
+        flush_interval 0.01s
+        chunk_limit_size 8m
+        retry_timeout 1s
+        retry_max_interval 1s
+      </buffer>
+    </store>
+  </match>
+</label>

data/test/plugin/docker/init.sql

@@ -0,0 +1,7 @@
+DROP TABLE IF EXISTS bulk;
+CREATE TABLE bulk (
+  col1 integer,
+  col2 text,
+  col3 float,
+  col4 jsonb
+);

data/test/plugin/test_out_postgres_bulk.rb

@@ -0,0 +1,75 @@
+require "helper"
+require "fluent/plugin/out_postgres_bulk.rb"
+
+class PostgresBulkOutputTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    $execed_params = []
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::PostgresBulkOutput) do
+      def client
+        Object.new.tap do |o|
+          def o.exec_params(*args)
+            $execed_params << args
+          end
+
+          def o.close
+          end
+        end
+      end
+    end.configure(conf)
+  end
+
+  def test_write
+    driver = create_driver %[
+      database db
+      username ksss
+      table public.test
+      column_names id,col1
+    ]
+    time = event_time
+    driver.run do
+      driver.feed("tag", time, {"id" => 1, "col1" => "foo"})
+      driver.feed("tag", time, {"id" => 1, "col1" => "bar"})
+      driver.feed("tag", time, {"id" => 1, "col1" => "baz"})
+    end
+    assert_equal($execed_params,
+      [
+        ["INSERT INTO public.test (id,col1) VALUES ($1,$2),($3,$4),($5,$6)",
+          [1, "foo", 1, "bar", 1, "baz"]]
+      ]
+    )
+  end
+
+  def test_write_huge
+    driver = create_driver %[
+      database db
+      username ksss
+      table public.test
+      column_names id,col1
+    ]
+    begin
+      orig = Fluent::Plugin::PostgresBulkOutput::N_PARAMETER_MAX
+      Fluent::Plugin::PostgresBulkOutput.__send__(:remove_const, :N_PARAMETER_MAX)
+      Fluent::Plugin::PostgresBulkOutput.__send__(:const_set, :N_PARAMETER_MAX, 10)
+      driver.run do
+        7.times do |i|
+          driver.feed("tag", Time.now.to_i, {"col1" => "foo", "id" => i})
+        end
+      end
+    ensure
+      Fluent::Plugin::PostgresBulkOutput.__send__(:remove_const, :N_PARAMETER_MAX)
+      Fluent::Plugin::PostgresBulkOutput.__send__(:const_set, :N_PARAMETER_MAX, orig)
+    end
+    assert_equal($execed_params,
+      [
+        ["INSERT INTO public.test (id,col1) VALUES ($1,$2),($3,$4),($5,$6),($7,$8),($9,$10)",
+          [0, "foo", 1, "foo", 2, "foo", 3, "foo", 4, "foo"]],
+        ["INSERT INTO public.test (id,col1) VALUES ($1,$2),($3,$4)",
+          [5, "foo", 6, "foo"]]
+      ]
+    )
+  end
+end

data/test/plugin/test_with_docker.rb

@@ -0,0 +1,89 @@
+return true unless ENV["WITH_DOCKER"]
+
+require "json"
+require "helper"
+require "fluent/plugin/out_postgres_bulk.rb"
+require 'fluent-logger'
+require 'pg'
+
+class WithDockerTest < Test::Unit::TestCase
+  def connect_pg(count = 1, limit = 10)
+    PG.connect(
+      host: URI.parse(ENV["DOCKER_HOST"]).hostname,
+      port: 5432,
+      dbname: "test",
+      user: "postgres",
+      password: "password",
+    ).tap { puts "connect_pg => ok" }
+  rescue PG::ConnectionBad
+    if count <= limit
+      puts "connect_pg => retry"
+      sleep 1
+      connect_pg(count + 1, limit)
+    else
+      raise
+    end
+  end
+
+  def connect_fluentd(count = 1, limit = 10)
+    Fluent::Logger.open(
+      Fluent::Logger::FluentLogger,
+      "test",
+      host: URI.parse(ENV["DOCKER_HOST"]).hostname,
+      port: 24224
+    )
+    if Fluent::Logger.default.connect?
+      puts "connect_fluentd => ok"
+    elsif count <= limit
+      puts "connect_fluentd => retry"
+      sleep 1
+      connect_fluentd(count + 1, limit)
+    else
+      raise "can not connect to fluent-logger"
+    end
+  end
+
+  setup do
+    system("cd #{__dir__}/docker; docker-compose up --build --detach")
+    at_exit { system("cd #{__dir__}/docker; docker-compose down") }
+    @client = connect_pg
+    @client.exec("truncate bulk")
+    connect_fluentd
+  end
+
+  teardown do
+    at_exit { @client.close }
+  end
+
+  def test_write
+    data = 3.times.map do |i|
+      {
+        "col1" => i,
+        "col2" => "$$$aaa$$$",
+        "col3" => nil,
+        "col4" => JSON.dump({"hello" => "world's end --","value" => i}),
+      }
+    end
+    data.each do |payload|
+      ret = Fluent::Logger.post_with_time(
+        "tag1",
+        payload,
+        Time.now.to_i
+      )
+      unless ret
+        raise "can not post to fluentd"
+      end
+    end
+    sleep 2 # wait flush
+    @client.exec("select * from bulk") do |r|
+      assert_equal(
+        r.to_a,
+        [
+          {"col1"=>"0", "col2"=>"$$$aaa$$$", "col3"=>nil, "col4"=>"{\"hello\": \"world's end --\", \"value\": 0}"},
+          {"col1"=>"1", "col2"=>"$$$aaa$$$", "col3"=>nil, "col4"=>"{\"hello\": \"world's end --\", \"value\": 1}"},
+          {"col1"=>"2", "col2"=>"$$$aaa$$$", "col3"=>nil, "col4"=>"{\"hello\": \"world's end --\", \"value\": 2}"}
+        ]
+      )
+    end
+  end
+end

metadata

@@ -0,0 +1,148 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-postgres-bulk
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- ksss
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-05-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.8
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.8
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: fluent plugin for bulk insert to postgres
+email:
+- co000ri@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- fluent-plugin-postgres-bulk.gemspec
+- lib/fluent/plugin/out_postgres_bulk.rb
+- test/helper.rb
+- test/plugin/docker/Dockerfile-fluentd
+- test/plugin/docker/Dockerfile-postgres
+- test/plugin/docker/Gemfile
+- test/plugin/docker/Gemfile.lock
+- test/plugin/docker/docker-compose.yml
+- test/plugin/docker/fluent.conf
+- test/plugin/docker/init.sql
+- test/plugin/docker/plugins/.gitkeep
+- test/plugin/test_out_postgres_bulk.rb
+- test/plugin/test_with_docker.rb
+homepage: https://github.com/ksss/fluent-plugin-postgres-bulk
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: fluent plugin for bulk insert to postgres
+test_files:
+- test/helper.rb
+- test/plugin/docker/Dockerfile-fluentd
+- test/plugin/docker/Dockerfile-postgres
+- test/plugin/docker/Gemfile
+- test/plugin/docker/Gemfile.lock
+- test/plugin/docker/docker-compose.yml
+- test/plugin/docker/fluent.conf
+- test/plugin/docker/init.sql
+- test/plugin/docker/plugins/.gitkeep
+- test/plugin/test_out_postgres_bulk.rb
+- test/plugin/test_with_docker.rb