replicator 0.1.0

data/.gitignore

@@ -0,0 +1,3 @@
+pkg
+*~
+Gemfile.lock

data/.rbenv-gemsets

@@ -0,0 +1 @@
+replicator

data/.rbenv-version

@@ -0,0 +1 @@
+ree-1.8.7-2012.02

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://www.rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Justin Balthrop
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,47 @@
+= Replicator
+
+Easy Postgres replication for Rails. Replicates specific columns from one table to
+another using auto-generated triggers. Just add simple directives in your migrations.
+
+== Usage:
+
+  ActiveRecord::Migration.extend(Replicator)
+
+  class AddReplication < ActiveRecord::Migration
+    def self.up
+      replicate :names,
+        :to     => :users,
+        :fields => [:first_name, :last_name]
+
+      replicate :data,
+        :to     => :users,
+        :fields => [:eye_color, :height]
+        :key    => 'user_id',
+        :prefix => 'data'
+
+      replicate :events,
+        :to         => :users,
+        :fields     => {:start_year => :year},
+        :key        => 'user_id',
+        :prefix     => 'type',
+        :prefix_map => {'BirthEvent' => 'birth', 'GraduationEvent' => 'grad'}
+
+      replicate :locations,
+        :to         => :users,
+        :fields     => [:latitude, :longitude, {[:city, :state, :country] => :location}],
+        :through    => 'events.address_id',
+        :key        => 'user_id',
+        :prefix     => 'events.type',
+        :prefix_map => {'BirthEvent' => 'birth', 'GraduationEvent' => 'grad'}
+    end
+  end
+ 
+There are a lot of options. The code is mostly self explanatory.
+
+== Install:
+
+  gem install replicator
+
+== License:
+
+Copyright (c) 2009 Justin Balthrop, Geni.com; Published under The MIT License, see License.txt

data/Rakefile

@@ -0,0 +1,10 @@
+require 'rake/testtask'
+require 'bundler/gem_tasks'
+
+Rake::TestTask.new do |t|
+  t.libs = ['lib']
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+task :default => :test

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/replicator.rb

@@ -0,0 +1,299 @@
+module Replicator
+  def replicate(table, opts)
+    action = opts.delete(:action) || :create
+    trigger = Trigger.new(table, opts)
+
+    case action
+    when :create
+      execute(trigger.create_sql)
+    when :drop
+      execute(trigger.drop_sql)
+    when :initialize     
+      sql_by_slice = trigger.initialize_sql
+      sql_by_slice.each do |slice, sql|
+        execute("DROP TABLE IF EXISTS #{slice[:name]}")
+        execute(sql)
+      end
+      replicated_table_slices(trigger.to).concat(sql_by_slice.keys)
+    else
+      raise "invalid action: #{action}"
+    end
+  end
+
+  def create_replicated_table(table_name)
+    table  = nil
+    fields = []
+    joins  = []
+
+    replicated_table_slices(table_name).each do |slice|
+      if table
+        joins << "LEFT OUTER JOIN #{slice[:name]} ON #{table}.id = #{slice[:name]}.id"
+      else
+        table = slice[:name]
+        fields << "#{table}.id"
+      end
+      fields << slice[:fields].collect {|f| "#{slice[:name]}.#{f}"}
+    end
+
+    execute %{
+      CREATE TABLE #{table_name} AS
+        SELECT #{fields.flatten.join(', ')}
+          FROM #{table}
+          #{joins.join(' ')}
+    }
+  end
+
+  def replicated_table_slices(table_name = nil)
+    if table_name
+      replicated_table_slices[table_name.to_sym] ||= []
+    else
+      @replicated_table_slices ||= {}
+    end
+  end
+
+  class Trigger
+    attr_reader :from, :to, :key, :through_table, :through_key, :condition, :prefix, :prefix_map, :dependent
+
+    def initialize(table, opts)
+      @from       = table
+      @to         = opts[:to]
+      @name       = opts[:name]
+      @key        = opts[:key] || 'id'
+      @condition  = opts[:condition] || opts[:if]
+      @prefix     = opts[:prefix]
+      @prefix_map = opts[:prefix_map]
+      @timestamps = opts[:timestamps]
+      @dependent  = opts[:dependent]
+
+      if opts[:through]
+        @through_table, @through_key = opts[:through].split('.')
+        raise "through must be of the form 'table.field'" unless @through_table and @through_key
+      end
+
+      # Use opts[:prefixes] to specify valid prefixes and use the identity mapping.
+      if @prefix_map.nil? and opts[:prefixes]
+        @prefix_map = {}
+        opts[:prefixes].each do |prefix|
+          @prefix_map[prefix] = prefix
+        end
+      end
+
+      if @prefix_map
+        @prefix, prefix_table = @prefix.split('.').reverse
+        
+        if prefix_table.nil? or prefix_table == from
+          @prefix_through = false
+        elsif prefix_table == through_table
+          @prefix_through = true
+        else
+          "unknown prefix table: #{prefix_table}" 
+        end
+      end
+
+      @fields = {}
+      opts[:fields] = [opts[:fields]] unless opts[:fields].kind_of?(Array)
+      opts[:fields].each do |field|
+        if field.kind_of?(Hash)
+          @fields.merge!(field)
+        else
+          @fields[field] = field
+        end
+      end
+    end
+
+    def fields(opts = nil)
+      if opts
+        opts[:row] ||= 'ROW'
+        # Add the prefixes and return an array of arrays.
+        @fields.collect do |from_field, to_field|
+          if from_field.kind_of?(Array)
+            from_field = from_field.collect {|f| "COALESCE(#{opts[:row]}.#{f}, '')"}.join(" || ' ' || ") 
+          else
+            from_field = "#{opts[:row]}.#{from_field}"
+          end
+          to_field = [opts[:prefix], to_field].compact.join('_')
+          [from_field, to_field]
+        end
+      else
+        # Just return the hash.
+        @fields
+      end
+    end
+
+    def create_sql
+      %{
+        CREATE OR REPLACE FUNCTION #{name}() RETURNS TRIGGER AS $$
+          DECLARE
+            ROW     RECORD;
+            THROUGH RECORD;
+          BEGIN
+            IF (TG_OP = 'DELETE') THEN
+              ROW := OLD;
+            ELSE
+              ROW := NEW;
+            END IF;
+            #{loop_sql}
+              IF #{conditions_sql} THEN
+                IF (TG_OP = 'DELETE') THEN
+                  #{delete_sql(:indent => 18)}
+                ELSE
+                  IF COUNT(*) = 0 FROM #{to} WHERE id = #{primary_key} THEN
+                    #{insert_sql}
+                  END IF;
+                  #{update_all_sql(:indent => 18)}
+                END IF;
+              END IF;
+            #{end_loop_sql}
+            RETURN NULL;
+          END;
+        $$ LANGUAGE plpgsql;
+        CREATE TRIGGER #{name} AFTER INSERT OR UPDATE OR DELETE ON #{from}
+          FOR EACH ROW EXECUTE PROCEDURE #{name}();
+      }
+    end
+
+    def initialize_sql
+      sql_by_slice = {}
+      
+      if through?
+        tables = "#{from}, #{through_table}"
+        join   = "#{from}.id = #{through_table}.#{through_key}"
+      else
+        tables = from
+      end
+
+      if prefix_map
+        prefix_map.each do |prefix_value, mapping|
+          slice_fields = []
+          field_sql = fields(:prefix => mapping, :row => from).collect do |from_field, to_field|
+            slice_fields << to_field
+            "#{from_field} AS #{to_field}"
+          end.join(', ')
+          where_sql  = "WHERE " << [join, "#{prefix_field(true)} = '#{prefix_value}'"].compact.join(' AND ')
+          table_name = "#{name}_#{mapping}"
+          
+          slice = {:name => table_name, :fields => slice_fields}
+          sql_by_slice[slice] = %{
+            CREATE TABLE #{table_name} AS
+              SELECT #{primary_key(true)} AS id, #{field_sql} FROM #{tables} #{where_sql}
+          }
+        end
+      else
+        slice_fields = []
+        field_sql = fields(:prefix => prefix, :row => from).collect do |from_field, to_field|
+          slice_fields << to_field
+          "#{from_field} AS #{to_field}"
+        end.join(', ')
+        where_sql  = "WHERE #{join}" if join
+        
+        slice = {:name => name, :fields => slice_fields}
+        sql_by_slice[slice] = %{
+          CREATE TABLE #{name} AS
+            SELECT #{primary_key(true)} AS id, #{field_sql} FROM #{tables} #{where_sql}
+        }
+      end
+      sql_by_slice
+    end
+
+    def drop_sql
+      "DROP FUNCTION IF EXISTS #{name}() CASCADE"
+    end
+
+    def timestamps?
+      @timestamps
+    end
+
+    def through?
+      not through_table.nil?
+    end
+
+    def prefix_through?
+      @prefix_through
+    end
+
+  private
+
+    def prefix_field(initialize = false)
+      if initialize
+        "#{prefix_through? ? through_table : from}.#{prefix}"
+      else
+        "#{prefix_through? ? 'THROUGH' : 'ROW'}.#{prefix}"
+      end
+    end
+
+    def primary_key(initialize = false)
+      if initialize
+        "#{through? ? through_table : from}.#{key}"
+      else
+        @primary_key ||= "#{through? ? 'THROUGH' : 'ROW'}.#{key}"
+      end
+    end      
+    
+    def name
+      @name ||= "replicate_#{from}_to_#{to}"
+    end
+
+    def loop_sql
+      "FOR THROUGH IN SELECT * FROM #{through_table} WHERE #{through_key} = ROW.id LOOP" if through?
+    end
+
+    def end_loop_sql
+      "END LOOP;" if through?
+    end
+
+    def conditions_sql
+      conditions = []
+      conditions << "#{primary_key} IS NOT NULL"
+      conditions << "#{prefix_field} IN (#{prefixes_sql})" if prefix_map 
+      conditions << condition if condition
+      conditions.join(' AND ')
+    end
+
+    def prefixes_sql
+      prefix_map.keys.collect {|p| "'#{p}'"}.join(',')
+    end
+
+    def insert_sql
+      if timestamps?
+        "INSERT INTO #{to} (id, created_at) VALUES (#{primary_key}, NOW());"
+      else
+        "INSERT INTO #{to} (id) VALUES (#{primary_key});"
+      end
+    end
+      
+    def update_sql(opts = {})
+      updates = fields(:prefix => opts[:prefix]).collect do |from_field, to_field|
+        from_field = 'NULL' if opts[:clear]
+        "#{to_field} = #{from_field}"
+      end
+      updates << "updated_at = NOW()" if timestamps?
+      sql = "SET #{updates.join(', ')}"
+      sql = "UPDATE #{to} #{sql} WHERE #{to}.id = #{primary_key};"
+      sql
+    end
+
+    def update_all_sql(opts = {})
+      return update_sql(opts.merge(:prefix => prefix)) unless prefix_map
+
+      sql = ''
+      opts[:indent] ||= 0
+      newline = "\n#{' ' * opts[:indent]}"
+      cond = 'IF'
+      prefix_map.each do |prefix_value, mapping|
+        sql << "#{cond} #{prefix_field} = '#{prefix_value}' THEN" + newline
+        sql << "  #{update_sql(opts.merge(:prefix => mapping))}" + newline 
+        cond = 'ELSIF'
+      end
+      sql << "END IF;"
+      sql
+    end
+
+    def delete_sql(opts = {})
+      if dependent == :destroy
+        "DELETE FROM #{to} WHERE id = #{primary_key};"
+      elsif dependent == :nullify
+        update_all_sql(:indent => opts[:indent], :clear => true)
+      end
+    end
+  end
+end

data/replicate.gemspec

@@ -0,0 +1,24 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |gem|
+  gem.name          = "replicator"
+  gem.version       = IO.read('VERSION')
+  gem.authors       = ["Justin Balthrop"]
+  gem.email         = ["git@justinbalthrop.com"]
+  gem.description   = %q{Easy Postgres replication for Rails}
+  gem.summary       = gem.description
+  gem.homepage      = "https://github.com/ninjudd/replicator"
+
+  gem.add_development_dependency 'shoulda', '3.0.1'
+  gem.add_development_dependency 'mocha'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'activerecord-postgresql-adapter'
+
+  gem.add_dependency 'activerecord',  '~> 2.3.9'
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+end

data/test/replicator_test.rb

@@ -0,0 +1,24 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class ReplicatorTest < Test::Unit::TestCase
+  def test_sql_update_all
+    t = Replicator::Trigger.new :locations,
+      :to         => :users,
+      :fields     => [:latitude, :longitude, {[:city, :state, :country] => :location}],
+      :through    => 'events.address_id',
+      :key        => 'user_id',
+      :prefix     => 'events.type',
+      :prefix_map => {'BirthEvent' => 'birth', 'GraduationEvent' => 'grad'}
+
+    sql = t.create_sql
+    # This is not a good to test this. I should actually add the triggers and check that they work.
+    assert_match 'FOR THROUGH IN SELECT * FROM events WHERE address_id = ROW.id LOOP', sql
+    assert_match 'INSERT INTO users (id) VALUES (THROUGH.user_id)', sql
+    ['birth', 'grad'].each do |prefix|
+      city_state_country = ['city', 'state', 'country'].map {|f| "COALESCE(ROW.#{f}, '')"}.join(" || ' ' || ")
+      assert_match "#{prefix}_location = " + city_state_country, sql
+      assert_match "#{prefix}_latitude = ROW.latitude", sql
+      assert_match "#{prefix}_longitude = ROW.longitude", sql
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,6 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+require 'mocha/setup'
+
+require 'replicator'

metadata

@@ -0,0 +1,153 @@
+--- !ruby/object:Gem::Specification 
+name: replicator
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Justin Balthrop
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-06-03 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: shoulda
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 3
+        - 0
+        - 1
+        version: 3.0.1
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: mocha
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: activerecord-postgresql-adapter
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 17
+        segments: 
+        - 2
+        - 3
+        - 9
+        version: 2.3.9
+  type: :runtime
+  version_requirements: *id005
+description: Easy Postgres replication for Rails
+email: 
+- git@justinbalthrop.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- .rbenv-gemsets
+- .rbenv-version
+- Gemfile
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/replicator.rb
+- replicate.gemspec
+- test/replicator_test.rb
+- test/test_helper.rb
+has_rdoc: true
+homepage: https://github.com/ninjudd/replicator
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.5.2
+signing_key: 
+specification_version: 3
+summary: Easy Postgres replication for Rails
+test_files: 
+- test/replicator_test.rb
+- test/test_helper.rb