fluent-plugin-postgres-replicator 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e0800a5552e7c84208560aa4e71ae507af0c826a
-  data.tar.gz: 0b17c898e047944522ef5f97fb6f10c3f4631701
+  metadata.gz: 8e07e9cc31d9517ca679f7d82b4e64c497a788c2
+  data.tar.gz: 234391df516fbe3a7b57c66c13b28d231d85d752
 SHA512:
-  metadata.gz: ce731ae495459c72229c84760df4c76963b08618f0d55300156b7d8303a838b0b8609d2b6d2ece8e27e0d01273eedd9b195bef6c74c215311b9cbf3149cf3d22
-  data.tar.gz: 3c98a0b8018703aa6e748c5b62eae1ddca8f4fe951c0c430e7594e5624d81b726ac1307766550d27b3b724eb274f0ce7ac0d838d11cee04a5803c9f89ffc7379
+  metadata.gz: 874a6e13a95cd16be2549b686db73c8bb00f0f5c0ec6c609828203cb38ffc585bb8e24c142f0b98640d205cac3335f4560ff32952aeb7b18df5a733cb2849463
+  data.tar.gz: d62069b79b4744282714d04361cf7f44ae38132af5677f35683449c0a64454f405d26b8fcbaefa389e99c87935b3599e22ff7df19efc61d33a30121a10d4afc9

data/.travis.yml

@@ -1,10 +1,10 @@
 language: ruby
 
 rvm:
-  - 1.9.3
-  - 2.0.0
   - 2.1
   - 2.2
+  - 2.3
+  - 2.4.1
 
 addons:
   postgresql: "9.4"

data/README.md

@@ -5,28 +5,31 @@
 
 Fluentd input plugin to track insert/update event from PostgreSQL.
 
+## Requirements
+
+| fluent-plugin-postgres-replicator | fluentd | ruby |
+|-----------------------------------|---------|------|
+| >= 0.1.0 | >= v0.14.0 | >= 2.1 |
+|  < 0.1.0 | >= v0.12.0 | >= 1.9 |
+
 ## Installation
 
 ```sh
-# requirements
 $ apt-get install libpq-dev
 
 $ gem install fluent-plugin-postgres-replicator
 ```
 
-## Usage
-
-In your Fluentd configuration, use `type postgres_replicator`.  
-Default values would look like this:
+## Configuration
 
 ```
 <source>
-  type postgres_replicator
+  @type postgres_replicator
   host localhost
   username pipeline
   password pipeline
   database pipeline
-  query SELECT hour, project, total_pages from wiki_stats;
+  sql SELECT hour, project, total_pages from wiki_stats;
   primary_keys hour,project
   interval 1m
   tag replicator.pipeline.wiki_stats.${event}.${primary_keys}
@@ -37,6 +40,18 @@ Default values would look like this:
 
 Bug reports and pull requests are welcome.
 
+You can run the test on your machine as below.
+
+```console
+$ docker run --name fluent-plugin-postgres-replicator-testing -e POSTGRES_DB=pg_repli_test_db -p 5432:5432 -d postgres:9.4.12-alpine
+$ psql -c 'create table pg_repli_test_table (id int8 primary key, total int8) ;' -U postgres -h 127.0.0.1 -d pg_repli_test_db
+$ psql -c 'insert into pg_repli_test_table values (1, 10) ;' -U postgres -h 127.0.0.1 -d pg_repli_test_db
+$ psql -c 'insert into pg_repli_test_table values (2, 20) ;' -U postgres -h 127.0.0.1 -d pg_repli_test_db
+
+$ bundle install --path vendor/bundle
+$ bundle exec rake test
+```
+
 ## License
 
 - Copyright (c) 2016 innossh

data/fluent-plugin-postgres-replicator.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-postgres-replicator"
-  gem.version       = "0.0.2"
+  gem.version       = "0.1.0"
   gem.authors       = ["innossh"]
   gem.email         = ["innossh@users.noreply.github.com"]
 
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "fluentd", [">= 0.14.0", "< 2"]
   gem.add_runtime_dependency "pg"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "test-unit"

data/lib/fluent/plugin/in_postgres_replicator.rb

@@ -1,126 +1,125 @@
-class Fluent::PostgresReplicatorInput < Fluent::Input
-  Fluent::Plugin.register_input('postgres_replicator', self)
-
-  config_param :host, :string, :default => 'localhost'
-  config_param :port, :integer, :default => 5432
-  config_param :username, :string, :default => 'root'
-  config_param :password, :string, :default => nil, :secret => true
-  config_param :database, :string, :default => nil
-  config_param :sql, :string, :default => nil
-  config_param :primary_keys, :string, :default => nil
-  config_param :interval, :string, :default => '10s'
-  config_param :tag, :string, :default => nil
-
-  def initialize
-    super
-    require 'pg'
-    require 'digest/sha1'
-  end
+require 'fluent/plugin/input'
+require 'pg'
+require 'digest/sha1'
+
+module Fluent::Plugin
+  class PostgresReplicatorInput < Input
+    Fluent::Plugin.register_input('postgres_replicator', self)
+
+    config_param :host, :string, :default => 'localhost'
+    config_param :port, :integer, :default => 5432
+    config_param :username, :string, :default => 'root'
+    config_param :password, :string, :default => nil, :secret => true
+    config_param :database, :string, :default => nil
+    config_param :sql, :string, :default => nil
+    config_param :primary_keys, :string
+    config_param :interval, :string, :default => '10s'
+    config_param :tag, :string
+
+    def configure(conf)
+      super
 
-  def configure(conf)
-    super
-    @interval = Fluent::Config.time_value(@interval)
-    if @primary_keys.nil?
-      raise Fluent::ConfigError, "primary_keys MUST be specified"
+      @interval = Fluent::Config.time_value(@interval)
+      @primary_keys = @primary_keys.split(/\s*,\s*/)
     end
-    if @tag.nil?
-      raise Fluent::ConfigError, "tag MUST be specified"
+
+    def start
+      super
+
+      @thread = Thread.new(&method(:run))
     end
-    @primary_keys = @primary_keys.split(/\s*,\s*/)
-  end
 
-  def start
-    @thread = Thread.new(&method(:run))
-  end
+    def shutdown
+      Thread.kill(@thread)
 
-  def shutdown
-    Thread.kill(@thread)
-  end
+      super
+    end
 
-  def run
-    begin
-      poll
-    rescue StandardError => e
-      log.error "failed to execute query. error: #{e.message}"
-      log.error e.backtrace.join("\n")
+    def run
+      begin
+        poll
+      rescue StandardError => e
+        log.error "failed to execute query. error: #{e.message}"
+        log.error e.backtrace.join("\n")
+      end
     end
-  end
 
-  def poll
-    hash_values = Hash.new
-    conn = get_connection()
-    loop do
-      rows_count = 0
-      start_time = Time.now
-      rows, conn = query(@sql, conn)
-      rows.each do |row|
-        row_ids = Array.new
-        @primary_keys.each do |primary_key|
-          if !row[primary_key].nil?
-            row_ids << row[primary_key]
+    def poll
+      hash_values = Hash.new
+      conn = get_connection()
+      loop do
+        rows_count = 0
+        start_time = Time.now
+        rows, conn = query(@sql, conn)
+        rows.each do |row|
+          row_ids = Array.new
+          @primary_keys.each do |primary_key|
+            if !row[primary_key].nil?
+              row_ids << row[primary_key]
+            end
+          end
+          if row_ids.size != @primary_keys.size
+            log.error "primary_keys column value is something wrong. :tag=>#{@tag} :primary_keys=>#{@primary_keys}"
+            break
           end
-        end
-        if row_ids.size != @primary_keys.size
-          log.error "primary_keys column value is something wrong. :tag=>#{@tag} :primary_keys=>#{@primary_keys}"
-          break
-        end
 
-        hash_value_id = row_ids.join('_')
-        hash_value = Digest::SHA1.hexdigest(row.flatten.join)
-        if !hash_values.include?(hash_value_id)
-          tag = format_tag(@tag, {:event => :insert})
-          emit_record(tag, row)
-        elsif hash_values[hash_value_id] != hash_value
-          tag = format_tag(@tag, {:event => :update})
-          emit_record(tag, row)
+          hash_value_id = row_ids.join('_')
+          hash_value = Digest::SHA1.hexdigest(row.flatten.join)
+          if !hash_values.include?(hash_value_id)
+            tag = format_tag(@tag, {:event => :insert})
+            emit_record(tag, row)
+          elsif hash_values[hash_value_id] != hash_value
+            tag = format_tag(@tag, {:event => :update})
+            emit_record(tag, row)
+          end
+          hash_values[hash_value_id] = hash_value
+          rows_count += 1
         end
-        hash_values[hash_value_id] = hash_value
-        rows_count += 1
+        conn.close
+        elapsed_time = sprintf('%0.02f', Time.now - start_time)
+        log.info "success to execute replicator. :tag=>#{@tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
+        sleep @interval
       end
-      conn.close
-      elapsed_time = sprintf('%0.02f', Time.now - start_time)
-      log.info "success to execute replicator. :tag=>#{@tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
-      sleep @interval
-    end
 
-  end
+    end
 
-  def query(sql, conn = nil)
-    begin
-      conn = (conn.nil? || conn.finished?) ? get_connection : conn
-      return conn.query(sql), conn
-    rescue Exception => e
-      log.warn "failed to execute query and will retry. error: #{e}"
-      sleep @interval
-      retry
+    def query(sql, conn = nil)
+      begin
+        conn = (conn.nil? || conn.finished?) ? get_connection : conn
+        return conn.query(sql), conn
+      rescue Exception => e
+        log.warn "failed to execute query and will retry. error: #{e}"
+        sleep @interval
+        retry
+      end
     end
-  end
 
-  def format_tag(tag, param)
-    pattern = {'${event}' => param[:event].to_s, '${primary_keys}' => @primary_keys.join('_')}
-    tag.gsub(/(\${[a-z_]+})/) do
-      log.warn "placeholder value is not found. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
-      pattern[$1]
+    def format_tag(tag, param)
+      pattern = {'${event}' => param[:event].to_s, '${primary_keys}' => @primary_keys.join('_')}
+      tag.gsub(/(\${[a-z_]+})/) do
+        log.warn "placeholder value is not found. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
+        pattern[$1]
+      end
     end
-  end
 
-  def emit_record(tag, record)
-    router.emit(tag, Fluent::Engine.now, record)
-  end
+    def emit_record(tag, record)
+      router.emit(tag, Fluent::Engine.now, record)
+    end
 
-  def get_connection
-    begin
-      return PG::Connection.new({
-        :host => @host,
-        :port => @port,
-        :user => @username,
-        :password => @password,
-        :dbname => @database
-      })
-    rescue Exception => e
-      log.warn "failed to get connection and will retry. error: #{e}"
-      sleep @interval
-      retry
+    def get_connection
+      begin
+        return PG::Connection.new({
+          :host => @host,
+          :port => @port,
+          :user => @username,
+          :password => @password,
+          :dbname => @database
+        })
+      rescue Exception => e
+        log.warn "failed to get connection and will retry. error: #{e}"
+        sleep @interval
+        retry
+      end
     end
   end
 end

data/test/plugin/test_in_postgres_replicator.rb

@@ -1,5 +1,5 @@
-require 'test/unit'
 require 'fluent/test'
+require 'fluent/test/driver/input'
 require 'fluent/plugin/in_postgres_replicator'
 
 class PostgresReplicatorTest < Test::Unit::TestCase
@@ -20,7 +20,7 @@ class PostgresReplicatorTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::PostgresReplicatorInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::PostgresReplicatorInput).configure(conf)
   end
 
   def test_configure
@@ -43,14 +43,12 @@ class PostgresReplicatorTest < Test::Unit::TestCase
   def test_emit
     d = create_driver
 
-    d.run do
-      sleep 2
-    end
+    d.run(expect_records: 2, timeout: 2)
 
-    emits = d.emits
-    assert_equal true, emits.length > 0
-    assert_equal ['pgreplicator.pg_repli_test_db.pg_repli_test_table.insert.id', @time, {'id' => '1', 'total' => '10'}], emits[0]
-    assert_equal ['pgreplicator.pg_repli_test_db.pg_repli_test_table.insert.id', @time, {'id' => '2', 'total' => '20'}], emits[1]
+    events = d.events
+    assert_equal true, events.length == 2
+    assert_equal ['pgreplicator.pg_repli_test_db.pg_repli_test_table.insert.id', @time, {'id' => '1', 'total' => '10'}], events[0]
+    assert_equal ['pgreplicator.pg_repli_test_db.pg_repli_test_table.insert.id', @time, {'id' => '2', 'total' => '20'}], events[1]
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-postgres-replicator
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - innossh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-07 00:00:00.000000000 Z
+date: 2017-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,14 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -105,7 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: PostgreSQL replication input plugin for Fluent