kodama 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in kodama.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,5 @@
+guard 'rspec', :version => 2, :cli => '-c' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Yusuke Mito
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,154 @@
+# Kodama
+
+Kodama is a MySQL replication listener based on [ruby-binlog](https://bitbucket.org/winebarrel/ruby-binlog/overview).
+Kodama provides a simple DSL to easily write your own replication listener.
+
+## Features
+
+- Provides simple DSL for writing binlog event handlers
+- Automatically restarts from the saved binlog position
+- Attempts to reconnect to MySQL when the connection is somehow teminated
+
+These features allow developers to focus on writing their own replication logic rather than having to spend time figuring things out.
+
+## Kodama Benefits
+
+Kodama can be used to replicate MySQL updates to other data stores, arbitrary software or even a flat file. The sole purpose of Kodama is to provide a convenient way to reflect the database updates to other components in your system.
+
+- Replicate from MySQL to Postgres
+- Replicate between tables with different schema
+- Sync production data to development DB while masking privacy information
+- Realtime full text index update
+
+## Dependencies
+
+This gem links against MySQL's libreplication C shared library. You need to first install the [mysql-replication-listener](https://launchpad.net/mysql-replication-listener) package.
+
+But official repository has some bugs. It is recommended to use [winebarrel's patched version](https://bitbucket.org/winebarrel/ruby-binlog/downloads) (There are rpm package and homebrew formula).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'kodama'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install kodama
+
+### binlog_format
+
+It is recommended to set the mysqld binlog_format option to ``ROW``. This is because the ``ROW`` format allows Kodama to pickup every single updates made to the database.
+
+```sql
+SET GLOBAL binlog_format = 'ROW';
+```
+
+## Usage
+
+### Simple client
+
+```ruby
+require 'kodama'
+
+Kodama::Client.start(:host => '127.0.0.1', :username => 'user') do |c|
+  c.binlog_position_file = 'position.log'
+  c.log_level = :info # [:debug|:info|:warn|:error|:fatal]
+  c.connection_retry_limit = 100 # times
+  c.connection_retry_wait = 3 # second
+
+  c.on_query_event do |event|
+    p event.query
+  end
+
+  c.on_row_event do |event|
+    p event.rows
+  end
+end
+```
+
+### Replicate to redis
+
+```ruby
+require 'rubygems'
+require 'kodama'
+require 'thread'
+require 'json'
+require 'redis'
+
+class Worker
+  attr_accessor :queue
+  def initialize
+    @queue = Queue.new
+    @redis = Redis.new
+  end
+
+  def start
+    Thread.start do
+      loop do
+        event = @queue.pop
+        record_id = get_row(event)[0] # first column is id
+        @redis.set "#{event.table_name}_#{record_id}", event.rows.to_json
+      end
+    end
+  end
+
+  def get_row(event)
+    case event.event_type
+    when /Write/, /Delete/
+      event.rows[0]    # [row]
+    when /Update/
+      event.rows[0][1] # [[old_row, new_row]]
+    end
+  end
+end
+
+
+worker = Worker.new
+worker.start
+
+Kodama::Client.start(:host => '127.0.0.1', :username => 'user') do |c|
+  c.binlog_position_file = 'position.log'
+
+  c.on_row_event do |event|
+    worker.queue << event
+  end
+end
+```
+
+## Configuration
+
+### binlog_position_file
+
+Sets the filename to save the binlog position.
+Kodama will read this file and resume listening from the stored position.
+
+### log_level
+
+Set logger's log level.
+It accepts ``:debug``, ``:info``, ``:warn``, ``:error``, ``:fatal``.
+
+### connection_retry_limit, connection_retry_wait
+
+If for some reason the connection to MySQL is terminated, Kodama will attempt to reconnect ``connection_retry_limit`` times, while waiting ``connection_retry_wait`` seconds between attempts.
+
+## Authors
+
+Yusuke Mito, Genki Sugawara
+
+## Based On
+
+- [ruby-binlog](https://bitbucket.org/winebarrel/ruby-binlog)
+- [mysql-replication-listener](https://launchpad.net/mysql-replication-listener)
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/kodama.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'kodama/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "kodama"
+  gem.version       = Kodama::VERSION
+  gem.authors       = ["Yusuke Mito"]
+  gem.email         = ["y310.1984@gmail.com"]
+  gem.description   = %q{ruby-binlog based MySQL replication listener}
+  gem.summary       = %q{ruby-binlog based MySQL replication listener}
+  gem.homepage      = "https://github.com/y310/kodama"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'ruby-binlog', '>= 0.1.8'
+
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'pry'
+  gem.add_development_dependency 'pry-nav'
+  gem.add_development_dependency 'guard-rspec', '2.1.2'
+  gem.add_development_dependency 'rb-fsevent', '~> 0.9.1'
+end

data/lib/kodama.rb

@@ -0,0 +1 @@
+require 'kodama/client'

data/lib/kodama/client.rb

@@ -0,0 +1,207 @@
+# coding: utf-8
+
+require 'binlog'
+require 'logger'
+
+module Kodama
+  class Client
+    LOG_LEVEL = {
+      :fatal => Logger::FATAL,
+      :error => Logger::ERROR,
+      :warn => Logger::WARN,
+      :info => Logger::INFO,
+      :debug => Logger::DEBUG,
+    }
+
+    class << self
+      def start(options = {}, &block)
+        client = self.new(mysql_url(options))
+        block.call(client)
+        client.start
+      end
+
+      def mysql_url(options = {})
+        password = options[:password] ? ":#{options[:password]}" : nil
+        port = options[:port] ? ":#{options[:port]}" : nil
+        "mysql://#{options[:username]}#{password}@#{options[:host]}#{port}"
+      end
+    end
+
+    def initialize(url)
+      @url = url
+      @binlog_info = BinlogInfo.new
+      @retry_info = RetryInfo.new(:limit => 100, :wait => 3)
+      @callbacks = {}
+      @logger = Logger.new(STDOUT)
+
+      self.log_level = :info
+    end
+
+    def on_query_event(&block); @callbacks[:on_query_event] = block; end
+    def on_rotate_event(&block); @callbacks[:on_rotate_event] = block; end
+    def on_int_var_event(&block); @callbacks[:on_int_var_event] = block; end
+    def on_user_var_event(&block); @callbacks[:on_user_var_event] = block; end
+    def on_format_event(&block); @callbacks[:on_format_event] = block; end
+    def on_xid(&block); @callbacks[:on_xid] = block; end
+    def on_table_map_event(&block); @callbacks[:on_table_map_event] = block; end
+    def on_row_event(&block); @callbacks[:on_row_event] = block; end
+    def on_incident_event(&block); @callbacks[:on_incident_event] = block; end
+    def on_unimplemented_event(&block); @callbacks[:on_unimplemented_event] = block; end
+
+    def binlog_position_file=(filename)
+      @position_file = position_file(filename)
+      @binlog_info.load!(@position_file)
+    end
+
+    def connection_retry_wait=(wait)
+      @retry_info.wait = wait
+    end
+
+    def connection_retry_limit=(limit)
+      @retry_info.limit = limit
+    end
+
+    def log_level=(level)
+      @logger.level = LOG_LEVEL[level]
+    end
+
+    def binlog_client(url)
+      Binlog::Client.new(url)
+    end
+
+    def position_file(filename)
+      PositionFile.new(filename)
+    end
+
+    def connection_retry_count
+      @retry_info.count
+    end
+
+    def start
+      begin
+        client = binlog_client(@url)
+        raise Binlog::Error, 'MySQL server has gone away' unless client.connect
+        @retry_info.count_reset
+
+        if @binlog_info.valid?
+          client.set_position(@binlog_info.filename, @binlog_info.position)
+        end
+
+        while event = client.wait_for_next_event
+          @binlog_info.position = event.next_position
+          case event
+          when Binlog::QueryEvent
+            callback :on_query_event, event
+            @binlog_info.save(@position_file)
+          when Binlog::RotateEvent
+            callback :on_rotate_event, event
+            @binlog_info.filename = event.binlog_file
+            @binlog_info.position = event.binlog_pos
+            @binlog_info.save(@position_file)
+          when Binlog::IntVarEvent
+            callback :on_int_var_event, event
+          when Binlog::UserVarEvent
+            callback :on_user_var_event, event
+          when Binlog::FormatEvent
+            callback :on_format_event, event
+          when Binlog::Xid
+            callback :on_xid, event
+          when Binlog::TableMapEvent
+            callback :on_table_map_event, event
+          when Binlog::RowEvent
+            callback :on_row_event, event
+            @binlog_info.save(@position_file)
+          when Binlog::IncidentEvent
+            callback :on_incident_event, event
+          when Binlog::UnimplementedEvent
+            callback :on_unimplemented_event, event
+          else
+            @logger.debug "Not Implemented: #{event.event_type}"
+          end
+        end
+      rescue Binlog::Error => e
+        @logger.debug e
+        if client.closed? && @retry_info.retryable?
+          sleep @retry_info.wait
+          @retry_info.count_up
+          retry
+        end
+        raise e
+      end
+    end
+
+    private
+    def callback(name, *args)
+      if @callbacks[name]
+        instance_exec *args, &@callbacks[name]
+      else
+        @logger.debug "Unhandled: #{name}"
+      end
+    end
+
+    class BinlogInfo
+      attr_accessor :filename, :position
+
+      def initialize(filename = nil, position = nil)
+        @filename = filename
+        @position = position
+      end
+
+      def valid?
+        @filename && @position
+      end
+
+      def save(position_file = nil)
+        if position_file
+          position_file.update(@filename, @position)
+        end
+      end
+
+      def load!(position_file)
+        @filename, @position = position_file.read
+      end
+    end
+
+    class RetryInfo
+      attr_accessor :count, :limit, :wait
+      def initialize(options = {})
+        @wait = options[:wait] || 3
+        @limit = options[:limit] || 100
+        @count = 0
+      end
+
+      def retryable?
+        @count < @limit
+      end
+
+      def count_up
+        @count += 1
+      end
+
+      def count_reset
+        @count = 0
+      end
+    end
+
+    class PositionFile
+      def initialize(filename)
+        @file = open(filename, File::RDWR|File::CREAT)
+        @file.sync = true
+      end
+
+      def update(filename, position)
+        @file.pos = 0
+        @file.write "#{filename}\t#{position}"
+        @file.truncate @file.pos
+      end
+
+      def read
+        @file.pos = 0
+        if line = @file.gets
+          filename, position = line.split("\t")
+          [filename, position.to_i]
+        end
+      end
+    end
+  end
+end

data/lib/kodama/version.rb

@@ -0,0 +1,3 @@
+module Kodama
+  VERSION = "0.0.1"
+end

data/spec/lib/client_spec.rb

@@ -0,0 +1,129 @@
+require 'spec_helper'
+
+describe Kodama::Client do
+  describe '.mysql_url' do
+    def mysql_url(options)
+      Kodama::Client.mysql_url(options)
+    end
+
+    it do
+      mysql_url(:username => 'user', :host => 'example.com').should == 'mysql://user@example.com'
+      mysql_url(:username => 'user', :host => 'example.com',
+                :password => 'password', :port => 3306).should == 'mysql://user:password@example.com:3306'
+    end
+  end
+
+  describe '#start' do
+    class TestBinlogClient
+      attr_accessor :connect
+      def initialize(events = [], connect = true)
+        @events = events
+        @connect = connect
+      end
+
+      def wait_for_next_event
+        event = @events.shift
+        stub_event(event)
+        event
+      end
+
+      def closed?
+        true
+      end
+
+      def stub_event(target_event)
+        target_event_class = target_event.instance_variable_get('@name')
+        [Binlog::QueryEvent, Binlog::RowEvent, Binlog::RotateEvent, Binlog::Xid].each do |event|
+          if event == target_event_class
+            event.stub(:===).and_return { true }
+          else
+            # :=== is stubbed
+            if event.method(:===).owner != Module
+              event.unstub(:===)
+            end
+          end
+        end
+      end
+    end
+
+    class TestPositionFile
+      def update(filename, position)
+      end
+
+      def read
+      end
+    end
+
+    def stub_binlog_client(events = [], connect = true)
+      client.stub(:binlog_client).and_return { TestBinlogClient.new(events, connect) }
+    end
+
+    def stub_position_file(position_file = nil)
+      client.stub(:position_file).and_return { position_file || TestPositionFile.new }
+    end
+
+    let(:client) { Kodama::Client.new('mysql://user@host') }
+
+    let(:rotate_event) do
+      mock(Binlog::RotateEvent).tap do |event|
+        event.stub(:next_position).and_return { 0 }
+        event.stub(:binlog_file).and_return { 'binlog' }
+        event.stub(:binlog_pos).and_return { 100 }
+      end
+    end
+
+    let(:query_event) do
+      mock(Binlog::QueryEvent).tap do |event|
+        event.stub(:next_position).and_return { 200 }
+      end
+    end
+
+    let(:row_event) do
+      mock(Binlog::RowEvent).tap do |event|
+        event.stub(:next_position).and_return { 300 }
+      end
+    end
+
+    let(:xid_event) do
+      mock(Binlog::Xid).tap do |event|
+        event.stub(:next_position).and_return { 400 }
+      end
+    end
+
+    it 'should receive query_event' do
+      stub_binlog_client([query_event])
+      expect {|block|
+        client.on_query_event(&block)
+        client.start
+      }.to yield_with_args(query_event)
+    end
+
+    it 'should receive row_event' do
+      stub_binlog_client([row_event])
+      expect {|block|
+        client.on_row_event(&block)
+        client.start
+      }.to yield_with_args(row_event)
+    end
+
+    it 'position is saved only on row, query and rotate event' do
+      stub_binlog_client([rotate_event, query_event, row_event, xid_event])
+      position_file = TestPositionFile.new.tap do |pf|
+        pf.should_receive(:update).with('binlog', 100).once.ordered
+        pf.should_receive(:update).with('binlog', 200).once.ordered
+        pf.should_receive(:update).with('binlog', 300).once.ordered
+      end
+      stub_position_file(position_file)
+      client.binlog_position_file = 'test'
+      client.start
+    end
+
+    it 'retry exactly specifeid times' do
+      stub_binlog_client([query_event], false)
+      client.connection_retry_limit = 2
+      client.connection_retry_wait = 0.1
+      expect { client.start }.to raise_error(Binlog::Error)
+      client.connection_retry_count.should == 2
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require 'kodama'

metadata

@@ -0,0 +1,181 @@
+--- !ruby/object:Gem::Specification 
+name: kodama
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Yusuke Mito
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-11-28 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: ruby-binlog
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 0
+        - 1
+        - 8
+        version: 0.1.8
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: pry
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: pry-nav
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: guard-rspec
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 2
+        - 1
+        - 2
+        version: 2.1.2
+  type: :development
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: rb-fsevent
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 57
+        segments: 
+        - 0
+        - 9
+        - 1
+        version: 0.9.1
+  type: :development
+  version_requirements: *id007
+description: ruby-binlog based MySQL replication listener
+email: 
+- y310.1984@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- kodama.gemspec
+- lib/kodama.rb
+- lib/kodama/client.rb
+- lib/kodama/version.rb
+- spec/lib/client_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/y310/kodama
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: ruby-binlog based MySQL replication listener
+test_files: 
+- spec/lib/client_spec.rb
+- spec/spec_helper.rb