fluent-plugin-pg-logical 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7f9e6ce82c9348118dd65343c808d07ca9a5a970
+  data.tar.gz: 5b658de92f88870b16136a19279f88dd013120b9
+SHA512:
+  metadata.gz: 1f2d70eda0cf9e4b111fa48820650b3f26ee9911a6f9e1f90858a282ff9c59021dad528e99cfd2142c1d1e4212112636c6ea4099510b6f7924d48009f593b0f6
+  data.tar.gz: 5491912d4db77d84a9564c85e471ee4c67aaa2cd402ebf0b215522e06685e5d3f50823abeed5bb0c40f8efb574ca69ba47d308742476eaebf808f8629ebd580e

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+vendor/*

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-mysql-replicator.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,14 @@
+Copyright (c) 2018- Masahiko Sawada
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

data/README.md

@@ -0,0 +1,110 @@
+# fluent-plugin-pg-logical
+
+## Overview
+
+Fluentd input plugin to track of changes (insert/update/delete) event on PostgreSQL using logical decoding.
+
+This plugin works as a WAL receiver of PostgreSQL and requires installation of logical decoding plugin to upstream PostgreSQL server.
+
+## Installation
+
+install with gem or fluent-gem command as:
+
+`````
+# for system installed fluentd
+$ gem install fluent-plugin-pg-logical
+`````
+
+## Configuration
+
+|Parameter|Type|Default|Remarks|
+|:--------|:---|:------|:----------|
+|host|string|'localhost'|-|
+|port|integer|5432|-|
+|user|string|'postgres'|-|
+|password|string|nil|-|
+|dbname|string|'postgres'|-|
+|slotanme|string|nil|Required|
+|plugin|string|nil|Required if 'create_slot' is specified|
+|status_interval|integer|10|Specifies the minimum frequency to send information about replication progress to upstream server|
+|tag|string|nil|-|
+|create_slot|bool|false|Specify to create the specified replication slot before start|
+|if_not_exists|bool|false|Do not error if slot already exists when creating a slot|
+
+## Restriction
+* Because logical decoding support only data changes (i.g. INSERT/UPDATE/DELETE), other changes such as DDL, sequence doesn't appear on fluentd input
+* Replication slots are reuiqred as much as you connect with fluent-plugin-pg-logical
+
+## Example with wal2json
+fluent-plugin-pg-logical requires a logical decoding plugin to get logical change set.This is a example of use of fluent-plugin-pg-logical with [wal2json](https://github.com/eulerto/wal2json), which decodes WAL to json object.
+
+1. Install wal2json to PostgreSQL
+Please refer to "Build and Install" section in wal2json documentation.
+
+2. Setting Configuration Parameters
+```
+<source>
+  @type pg_logical
+  host pgserver
+  port 5432
+  user postgres
+  dbname replication_db
+  slotname wal2json_slot
+  plugin wal2json
+  create_slot true
+  if_not_exists true
+</source>
+```
+
+3. Run fluentd
+Launch fluentd.
+
+4. Issue some SQL
+```sql
+=# CREATE TABLE hoge (c int primary key);
+CREATE TABLE
+=#INSERT INTO hoge VALUES (1), (2), (3);
+INSERT 0 3
+=# BEGIN;
+BEGIN
+=# UPDATE hoge SET c = c + 10 WHERE c = 1;
+UPDATE 1
+=# UPDATE hoge SET c = c + 20 WHERE c = 2;
+UPDATE 1
+=# COMMIT;
+COMMIT
+```
+
+You will get,
+
+```
+2018-02-03 16:02:20.073058428 +0900 : "{\"change\":[]}"
+2018-02-03 16:02:38.266394490 +0900 : "{\"change\":[{\"kind\":\"insert\",\"schema\":\"public\",\"table\":\"hoge\",\"columnnames\":[\"c\"],\"columntypes\":[\"integer\"],\"columnvalues\":[1]},{\"kind\":\"insert\",\"schema\":\"public\",\"table\":\"hoge\",\"columnnames\":[\"c\"],\"columntypes\":[\"integer\"],\"columnvalues\":[2]},{\"kind\":\"insert\",\"schema\":\"public\",\"table\":\"hoge\",\"columnnames\":[\"c\"],\"columntypes\":[\"integer\"],\"columnvalues\":[3]}]}"
+2018-02-03 16:03:05.890485185 +0900 : "{\"change\":[{\"kind\":\"update\",\"schema\":\"public\",\"table\":\"hoge\",\"columnnames\":[\"c\"],\"columntypes\":[\"integer\"],\"columnvalues\":[11],\"oldkeys\":{\"keynames\":[\"c\"],\"keytypes\":[\"integer\"],\"keyvalues\":[1]}},{\"kind\":\"update\",\"schema\":\"public\",\"table\":\"hoge\",\"columnnames\":[\"c\"],\"columntypes\":[\"integer\"],\"columnvalues\":[22],\"oldkeys\":{\"keynames\":[\"c\"],\"keytypes\":[\"integer\"],\"keyvalues\":[2]}}]}"
+```
+Because current (at least up to version 10) PostgreSQL doesn't support DDL replication, `CREATE TABLE` command doesn't appear to fluentd input.
+
+
+You can also monitor the activity of fluent-plugin-pg-logical on upstream server.
+
+```sql
+=# SELECT usename, application_name, sent_location, write_location, flush_location FROM pg_stat_replication ;
+
+ usename  | application_name | sent_location | write_location | flush_location 
+----------+------------------+---------------+----------------+----------------
+ masahiko | pg-logical       | 0/15ADD70     | 0/15ADAC8      | 0/15ADAC8
+(1 row)
+
+```
+
+## TODO
+* Add travis test
+* Table filtering
+
+## Copyright
+
+Copyright © 2018- Masahiko Sawada
+
+## License
+
+Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-pg-logical.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |s|
+  s.name        = "fluent-plugin-pg-logical"
+  s.version     = "0.0.1"
+  s.authors     = ["Masahiko Sawada"]
+  s.email       = ["sawada.mshk@gmail.com"]
+  s.homepage    = "https://github.com/MasahikoSawada/fluent-plugin-pg-logical"
+  s.summary     = %q{Fluentd input plugin to track of changes on PostgreSQL server using logical decoding}
+  s.license     = "Apache-2.0"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.required_ruby_version = "> 2.1"
+
+  s.add_development_dependency "rake"
+  s.add_development_dependency "webmock", "~> 1.24.0"
+  s.add_development_dependency "test-unit", ">= 3.1.0"
+
+  s.add_runtime_dependency "fluentd"
+  s.add_runtime_dependency "pg"
+end

data/lib/fluent/plugin/in_pg_logical.rb

@@ -0,0 +1,311 @@
+require 'fluent/input'
+
+module Fluent
+  class PgLogicalInput < Fluent::Input
+    # Register input plugin
+    Plugin.register_input( 'pg_logical', self)
+
+    def initialize
+      require 'pg'
+      super
+    end
+
+    config_param :host, :string, :default => 'localhost'
+    config_param :port, :integer, :default => 5432
+    config_param :user, :string, :default => 'postgres'
+    config_param :password, :string, :default => nil, :secret => true
+    config_param :dbname, :string, :default => 'postgres'
+    config_param :slotname, :string, :default => nil
+    config_param :plugin,   :string, :default => nil
+    config_param :status_interval, :integer, :default => 10
+    config_param :tag, :string, :default => nil
+    config_param :create_slot, :bool, :default=> false
+    config_param :if_not_exists, :bool, :default => false
+    def configure(conf)
+      super
+
+      # 'slot_name' parameter is required.
+      if (@slotname.nil?)
+        raise Fluent::ConfigError, "pg-logical: missing 'slotname' parameter."
+      end
+
+      # If 'create_slot' parameter is specified, 'plugin' name is required.
+      if (!@create_slot.nil? and @plugin.nil?)
+        raise Fluent::ConfigError, "pg-logical: 'create_slot' parameter reuiqres to specify 'plugin' parameter."
+      end
+
+      log.info ":host=>#{host} :dbname=>#{dbname} :port=>#{port} :user=>#{user} :tag=>#{tag} :slotname=>#{slotname} :plugin=>#{plugin} :status_interval=>#{status_interval}"
+    end
+
+    def start
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      if (!@conn.nil?)
+        @conn.put_copy_end()
+        @conn.flush()
+      end
+
+      Thread.kill(@thread)
+    end
+
+    def run
+      begin
+        streamLogicalLog
+      rescue StandardError => e
+        log.error "pg_logical: failed to execute query."
+        log.error "error: #{e.message}"
+        log.error e.backtrace.join("\n")
+      end
+    end
+
+    # Start logical replication
+    def start_streaming
+      # Identify system, and get start lsn
+      res = @conn.exec("IDENTIFY_SYSTEM")
+      systemid = res.getvalue(0, 0)
+      tli = res.getvalue(0, 1)
+      xlogpos = res.getvalue(0, 2)
+      dbname = res.getvalue(0, 3)
+
+      # Start logical replication
+      strbuf = "START_REPLICATION SLOT %s LOGICAL %s" % [@slotname, xlogpos]
+      @conn.exec(strbuf)
+    end
+
+    # Get a connection
+    def get_connection
+      begin
+        return PG::connect(
+                           :host => @host,
+                           :port => @port,
+                           :user => @user,
+                           :password => @password,
+                           :dbname => @dbname,
+                           :application_name => 'pg-logical',
+                           :replication => "database"
+                         )
+      rescue Exception => e
+        log.warn "pg-logical: #{e}"
+        sleep 5
+        retry
+      end
+    end
+
+    # Main routine of pg-logical plugin. Stream logical WAL.
+    def streamLogicalLog
+      begin
+        @conn = get_connection()
+
+        # Create replication slot if required
+        create_replication_slot()
+
+        # Start replication
+        start_streaming()
+
+        record = nil
+        socket = @conn.socket_io
+        time_to_abort = false
+        last_status = Time.now
+        loop do
+          # Get current timestamp
+          now = Time.now
+
+          # Send feedback if necessary
+          last_status = sendFeedback(now, last_status, false)
+
+          # Get a decoded WAL decode
+          record = @conn.get_copy_data(true)
+
+          # In async mode, and no data available. We block on reading but
+          # not more than the specified timeout, so that we can send a
+          # response back to the client.# In asynchronou mode,
+          if (record == false)
+            # XXX: maybe better to use libev?
+            r = select([socket], [], [], 10.0)
+
+            if (r.nil?)
+              # Got a timeout or signal. Continue the loop and either
+              # deliver a status packet to the server or just go back into
+              # blocking.
+              next
+            end
+
+            # There is actual data on socket, consume it.
+            @conn.consume_input()
+            next
+          end
+
+          # record is nil means that copy is done.
+          if (record.nil?)
+            next
+          end
+
+          # Process a record, get extracted record
+          wal = extractRecord(record)
+
+          if (wal[:type] == 'w')	# WAL data
+            #log.info "[GET w] start : #{wal[:start_lsn]}, end : #{wal[:end_lsn]}, time : #{wal[:send_time]}, data : #{wal[:data]}"
+            last_status = sendFeedback(now, last_status, true)
+
+            @router.emit(@tag, Fluent::Engine.now, wal[:data])
+
+          elsif (wal[:type] == 'k') # Keepalive data
+            #log.info "[GET k] end : #{wal[:end_lsn]}, time : #{wal[:send_time]}, reply_required : #{wal[:reply_required]}"
+
+            if (wal[:reply_required] == 1)
+              last_status = sendFeedback(now, last_status, true)
+            end
+          end
+        end
+      rescue Exception => e
+        log.warn "pg-logical: #{e}"
+        sleep 5
+        retry
+      ensure
+        @conn.finish if !@conn.nil?
+      end
+    end
+
+    # Return extracted WAL data into a hash map
+    def extractRecord(record)
+      r = record.unpack("a")
+      wal = {}
+
+      if (r[0] == 'w') # WAL data
+        # -- WAL data format ------
+        # 1. 'w'	: byte
+        # 2. start_lsn	: uint64
+        # 3. end_lsn	: uint64
+        # 4. send_time	: uint64
+        # 5. data
+        # ------------------------
+        r = record.unpack("aNNNNNNc*")
+
+        start_lsn_h = r[1]
+        start_lsn_l = r[2]
+        end_lsn_h = r[3]
+        end_lsn_l = r[4]
+        send_time_h = r[5]
+        send_time_l = r[6]
+        data = r[7 .. r.size].pack("C*")
+
+        start_lsn = (start_lsn_h << 32) + start_lsn_l
+        end_lsn = (end_lsn_h << 32) + end_lsn_l
+        send_time = (send_time_h << 32) + send_time_l
+
+        wal[:type] = 'w'
+        wal[:start_lsn] = start_lsn
+        wal[:end_lsn] = end_lsn
+        wal[:send_time] = send_time
+        wal[:data] = data
+      elsif (r[0] == 'k') # keepalive message
+        # -- Keepalive format ------
+        # 1. 'k'	: byte
+        # 2. end_lsn	: uint64
+        # 3. send_time	: uint64
+        # 4. reply_required : byte
+        # ------------------------
+        r = record.unpack("aNNNNc")
+
+        end_lsn_h = r[1]
+        end_lsn_l = r[2]
+        send_time_h = r[3]
+        send_time_l = r[4]
+        reply_required = r[5]
+
+        end_lsn = (end_lsn_h << 32) + end_lsn_l
+        send_time = (send_time_h << 32) + send_time_l
+
+        wal[:type] = 'k'
+        wal[:end_lsn] = end_lsn
+        wal[:send_time] = send_time
+        wal[:reply_required] = reply_required
+      end
+
+      # Update reveive lsn
+      if (@recv_lsn.nil? or wal[:end_lsn] > @recv_lsn)
+        @recv_lsn = wal[:end_lsn]
+      end
+
+      return wal
+    end
+
+    # Return the last feedback time
+    def sendFeedback(now, last_status, force)
+
+      # If the user doesn't want status to be reported the
+      # upstream server, be sure to exit before doing anything
+      # at all.
+      if (!force and now - last_status < @status_interval)
+        return last_status
+      end
+
+      # Report current status to upstream server
+      if (!@recv_lsn.nil?)
+        # -- Feedback format ------
+        # 1. 'r'	: byte
+        # 2. write_lsn	: uint64
+        # 3. flush_lsn	: uint64
+        # 3. apply_lsn	: uint64
+        # 4. send_time	: uint64
+        # 5. reply_required : byte
+        # ------------------------
+        feedback_msg = ['r']
+
+        recv_lsn_h = @recv_lsn >> 32
+        recv_lsn_l = @recv_lsn & 0xFFFFFFFF
+
+        # write
+        feedback_msg.push(recv_lsn_h)
+        feedback_msg.push(recv_lsn_l)
+
+        # flush
+        feedback_msg.push(recv_lsn_h)
+        feedback_msg.push(recv_lsn_l)
+
+        # apply
+        feedback_msg.push(0)
+        feedback_msg.push(0)
+
+        # send_time
+        now_h = now.to_i >> 32
+        now_l = now.to_i & 0xFFFFFFFF
+        feedback_msg.push(now_h)
+        feedback_msg.push(now_l)
+
+        # Require reply
+        feedback_msg.push(0)
+        packed = feedback_msg.pack("aN8c")
+
+        @conn.flush
+        if (!@conn.put_copy_data(packed))
+          raise "error"
+        end
+
+        # Update last_status as we've sent
+        last_status = now
+      end
+
+      return last_status
+    end
+
+    # Create a replication slot
+    def create_replication_slot
+      begin
+        strbuf = "CREATE_REPLICATION_SLOT %s LOGICAL %s" % [@slotname, @plugin]
+        puts strbuf
+        @conn.exec(strbuf)
+      rescue PG::Error
+        # If if_not_exists is set, ignore the error
+        if (@if_not_exists)
+          log.info "pg-logical: could not create replication slot %s" % @slotname
+          return
+        end
+
+        log.error "pg-logical: could not create replication slot %s" % @slotname
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-pg-logical
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Masahiko Sawada
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-02-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.24.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.24.0
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- sawada.mshk@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-pg-logical.gemspec
+- lib/fluent/plugin/in_pg_logical.rb
+homepage: https://github.com/MasahikoSawada/fluent-plugin-pg-logical
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: '2.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: Fluentd input plugin to track of changes on PostgreSQL server using logical
+  decoding
+test_files: []