fluent-plugin-postgres-replicator 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3ca7d5dc0cc16563d50a913c50da4d89f130ceb3
-  data.tar.gz: a12eaccf0624403b32359673935b6614ed367363
+  metadata.gz: e0800a5552e7c84208560aa4e71ae507af0c826a
+  data.tar.gz: 0b17c898e047944522ef5f97fb6f10c3f4631701
 SHA512:
-  metadata.gz: c5d817d1e75134379f26443e0106b28a1afc2e03de7b94467cc8a21660169365a6295da941e6bad9cf25373e066acfd3ec7628ce09c13a4c1d1f3fe749fd846e
-  data.tar.gz: 13db91bfb686ed5eeec844d50a68eb55f72f7e82495223191f27b2d2f9ee2b92e65ac7bcaeaf3923f20b65f9416069a69eaf825c77677c8840e6ad869c0c6b4d
+  metadata.gz: ce731ae495459c72229c84760df4c76963b08618f0d55300156b7d8303a838b0b8609d2b6d2ece8e27e0d01273eedd9b195bef6c74c215311b9cbf3149cf3d22
+  data.tar.gz: 3c98a0b8018703aa6e748c5b62eae1ddca8f4fe951c0c430e7594e5624d81b726ac1307766550d27b3b724eb274f0ce7ac0d838d11cee04a5803c9f89ffc7379

data/.travis.yml

@@ -1,4 +1,19 @@
 language: ruby
+
 rvm:
-  - 2.2.3
-before_install: gem install bundler -v 1.10.6
+  - 1.9.3
+  - 2.0.0
+  - 2.1
+  - 2.2
+
+addons:
+  postgresql: "9.4"
+
+services:
+  - postgresql
+
+before_script:
+  - psql -c 'create database pg_repli_test_db;' -U postgres
+  - psql -c 'create table pg_repli_test_table (id int8 primary key, total int8) ;' -U postgres -d pg_repli_test_db
+  - psql -c 'insert into pg_repli_test_table values (1, 10) ;' -U postgres -d pg_repli_test_db
+  - psql -c 'insert into pg_repli_test_table values (2, 20) ;' -U postgres -d pg_repli_test_db

data/README.md

@@ -1,34 +1,37 @@
-# Fluent::Plugin::Postgres::Replicator
+# Fluent::Plugin::PostgresReplicator, a plugin for [Fluentd](http://www.fluentd.org)
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/fluent/plugin/postgres/replicator`. To experiment with that code, run `bin/console` for an interactive prompt.
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-postgres-replicator.svg)](https://badge.fury.io/rb/fluent-plugin-postgres-replicator)
+[![Build Status](https://travis-ci.org/innossh/fluent-plugin-postgres-replicator.svg?branch=master)](https://travis-ci.org/innossh/fluent-plugin-postgres-replicator)
 
-TODO: Delete this and the text above, and describe your gem
+Fluentd input plugin to track insert/update event from PostgreSQL.
 
 ## Installation
 
-Add this line to your application's Gemfile:
+```sh
+# requirements
+$ apt-get install libpq-dev
 
-```ruby
-gem 'fluent-plugin-postgres-replicator'
+$ gem install fluent-plugin-postgres-replicator
 ```
 
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install fluent-plugin-postgres-replicator
-
 ## Usage
 
-TODO: Write usage instructions here
+In your Fluentd configuration, use `type postgres_replicator`.  
+Default values would look like this:
 
-## Development
-
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake false` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
-
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+```
+<source>
+  type postgres_replicator
+  host localhost
+  username pipeline
+  password pipeline
+  database pipeline
+  query SELECT hour, project, total_pages from wiki_stats;
+  primary_keys hour,project
+  interval 1m
+  tag replicator.pipeline.wiki_stats.${event}.${primary_keys}
+</source>
+```
 
 ## Contributing
 
@@ -36,5 +39,5 @@ Bug reports and pull requests are welcome.
 
 ## License
 
-- Copyright (c) 2015 innossh
+- Copyright (c) 2016 innossh
 - [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)

data/fluent-plugin-postgres-replicator.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-postgres-replicator"
-  gem.version       = "0.0.1"
+  gem.version       = "0.0.2"
   gem.authors       = ["innossh"]
   gem.email         = ["innossh@users.noreply.github.com"]
 

data/lib/fluent/plugin/in_postgres_replicator.rb

@@ -6,7 +6,7 @@ class Fluent::PostgresReplicatorInput < Fluent::Input
   config_param :username, :string, :default => 'root'
   config_param :password, :string, :default => nil, :secret => true
   config_param :database, :string, :default => nil
-  config_param :query, :string, :default => nil
+  config_param :sql, :string, :default => nil
   config_param :primary_keys, :string, :default => nil
   config_param :interval, :string, :default => '10s'
   config_param :tag, :string, :default => nil
@@ -26,7 +26,7 @@ class Fluent::PostgresReplicatorInput < Fluent::Input
     if @tag.nil?
       raise Fluent::ConfigError, "tag MUST be specified"
     end
-    @primary_keys = @primary_keys.split(',')
+    @primary_keys = @primary_keys.split(/\s*,\s*/)
   end
 
   def start
@@ -41,8 +41,8 @@ class Fluent::PostgresReplicatorInput < Fluent::Input
     begin
       poll
     rescue StandardError => e
-      $log.error "failed to execute query. error: #{e.message}"
-      $log.error e.backtrace.join("\n")
+      log.error "failed to execute query. error: #{e.message}"
+      log.error e.backtrace.join("\n")
     end
   end
 
@@ -52,15 +52,17 @@ class Fluent::PostgresReplicatorInput < Fluent::Input
     loop do
       rows_count = 0
       start_time = Time.now
-      rows, conn = query(@query, conn)
+      rows, conn = query(@sql, conn)
       rows.each do |row|
         row_ids = Array.new
         @primary_keys.each do |primary_key|
-          if row[primary_key].nil?
-            $log.error "primary_key value is not found. :tag=>#{@tag} :primary_key=>#{primary_key}"
-            break
+          if !row[primary_key].nil?
+            row_ids << row[primary_key]
           end
-          row_ids << row[primary_key]
+        end
+        if row_ids.size != @primary_keys.size
+          log.error "primary_keys column value is something wrong. :tag=>#{@tag} :primary_keys=>#{@primary_keys}"
+          break
         end
 
         hash_value_id = row_ids.join('_')
@@ -77,18 +79,18 @@ class Fluent::PostgresReplicatorInput < Fluent::Input
       end
       conn.close
       elapsed_time = sprintf('%0.02f', Time.now - start_time)
-      $log.info "success to execute replicator. :tag=>#{@tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
+      log.info "success to execute replicator. :tag=>#{@tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
       sleep @interval
     end
 
   end
 
-  def query(query, conn = nil)
+  def query(sql, conn = nil)
     begin
       conn = (conn.nil? || conn.finished?) ? get_connection : conn
-      return conn.query(query), conn
+      return conn.query(sql), conn
     rescue Exception => e
-      $log.warn "failed to execute query and will retry. error: #{e}"
+      log.warn "failed to execute query and will retry. error: #{e}"
       sleep @interval
       retry
     end
@@ -97,7 +99,7 @@ class Fluent::PostgresReplicatorInput < Fluent::Input
   def format_tag(tag, param)
     pattern = {'${event}' => param[:event].to_s, '${primary_keys}' => @primary_keys.join('_')}
     tag.gsub(/(\${[a-z_]+})/) do
-      $log.warn "placeholder value is not found. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
+      log.warn "placeholder value is not found. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
       pattern[$1]
     end
   end
@@ -116,7 +118,7 @@ class Fluent::PostgresReplicatorInput < Fluent::Input
         :dbname => @database
       })
     rescue Exception => e
-      $log.warn "failed to get connection and will retry. error: #{e}"
+      log.warn "failed to get connection and will retry. error: #{e}"
       sleep @interval
       retry
     end

data/test/plugin/test_in_postgres_replicator.rb

@@ -5,13 +5,18 @@ require 'fluent/plugin/in_postgres_replicator'
 class PostgresReplicatorTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
+    @time = Fluent::Engine.now
   end
 
   CONFIG = %[
     host 127.0.0.1
     port 5432
+    username postgres
+    database pg_repli_test_db
+    sql SELECT id, total from pg_repli_test_table;
+    interval 1s
     primary_keys id
-    tag pgreplicator.db.table.${event}.${primary_keys}
+    tag pgreplicator.pg_repli_test_db.pg_repli_test_table.${event}.${primary_keys}
   ]
 
   def create_driver(conf = CONFIG)
@@ -26,8 +31,26 @@ class PostgresReplicatorTest < Test::Unit::TestCase
     d = create_driver
     assert_equal '127.0.0.1', d.instance.host
     assert_equal 5432, d.instance.port
-    assert_equal [ 'id' ], d.instance.primary_keys
-    assert_equal 'pgreplicator.db.table.${event}.${primary_keys}', d.instance.tag
+    assert_equal 'postgres', d.instance.username
+    assert_equal nil, d.instance.password
+    assert_equal 'pg_repli_test_db', d.instance.database
+    assert_equal 'SELECT id, total from pg_repli_test_table;', d.instance.sql
+    assert_equal 1, d.instance.interval
+    assert_equal ['id'], d.instance.primary_keys
+    assert_equal 'pgreplicator.pg_repli_test_db.pg_repli_test_table.${event}.${primary_keys}', d.instance.tag
+  end
+
+  def test_emit
+    d = create_driver
+
+    d.run do
+      sleep 2
+    end
+
+    emits = d.emits
+    assert_equal true, emits.length > 0
+    assert_equal ['pgreplicator.pg_repli_test_db.pg_repli_test_table.insert.id', @time, {'id' => '1', 'total' => '10'}], emits[0]
+    assert_equal ['pgreplicator.pg_repli_test_db.pg_repli_test_table.insert.id', @time, {'id' => '2', 'total' => '20'}], emits[1]
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-postgres-replicator
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - innossh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-06 00:00:00.000000000 Z
+date: 2016-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd