my_stuff-multidb 0.0.1

data/COPYING

@@ -0,0 +1,13 @@
+Copyright (c) 2011-present, Fred Emmott <copyright@fredemmott.co.uk>
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

data/README.rdoc

@@ -0,0 +1,65 @@
+= Overview
+
+This provides an API that's relatively convenient for:
+* Reading from slaves
+* Using multiple shards
+
+= Usage
+
+ module MySpecProvider
+   def spec_for_new
+     { :adapter => 'mysql', :host => ...}
+   end
+
+   def spec_for_master shard_id
+     ...
+   end
+
+   def spec_for_slave shard_id
+     ...
+   end
+ end
+
+ module MyDB
+   class Widget < ActiveRecord::Base; end
+   include MyStuff::MultiDB::Sharded
+   extend MySpecProvider
+ end
+
+ MyDB.with_master_for_new do |db,spec|
+   db::Widget.create(...)
+ end
+
+ MyDB.with_slave_for(shard_id) do |db|
+   db::Widget.find(record_id);
+   ...
+ end
+
+Another option, if you only do primary key lookups, is to encode the shard
+ID into the record id.
+
+For more, see:
+
+* ./examples/run
+* comments at top of lib/mystuff/multidb.rb
+
+= How It Works
+
+For every spec, it defines a new sub-module of MyStuff::MultiDB, which
+encodes the database details, and creates new subclasses of your
+ActiveRecord::Base classes, within these modules. For example:
+
+ $ bin/my_stuff-multidb-unmangle MyStuff::MultiDB::MYSTUFF_MULTIDB_DB_216c6f63616c686f7374213333303621::MyDB::Widget
+ MyStuff::MultiDB::<localhost:3306/>::MyDB::Widget
+ $
+
+There's also lots of deep voodoo making the ActiveRecord stack use the right
+connection details :)
+
+= Caveats
+
+You can not do cross-shard operations simply, eg joins.
+
+= Copying
+
+See the COPYING file.

data/bin/my_stuff-multidb-unmangle

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+# Copyright 2011-present Fred Emmott. See COPYING file.
+
+_FILE = File.readlink(__FILE__) rescue __FILE__
+_DIR = File.expand_path(File.dirname(_FILE))
+$LOAD_PATH.push(File.expand_path(_DIR +  '/../lib'))
+
+require 'my_stuff/multidb'
+
+def filter line
+  line.gsub(/MYSTUFF_MULTIDB_DB_[a-z0-9]+/){ |mangled|
+    data = MyStuff::MultiDB.unmangle(mangled)
+    "<%s:%d/%s>" % [
+      data[:host],
+      data[:port],
+      data[:database],
+    ]
+  }
+end
+
+if ARGV.empty?
+  STDIN.each_line do |line|
+    puts filter(line.strip)
+  end
+else
+    puts filter(ARGV.first)
+end

data/lib/my_stuff/multidb.rb

@@ -0,0 +1,180 @@
+# Copyright 2011-present Fred Emmott. See COPYING file.
+
+require 'base64'
+
+module MyStuff
+  # =Example
+  #
+  #  module Foo
+  #    class SomeTable < ActiveRecord::Base
+  #    end
+  #    include MyStuff::MultiDB::Unsharded
+  #  end
+  #  module Bar
+  #    class SomeTable < ActiveRecord::Base
+  #    end
+  #    include MyStuff::MultiDB::Sharded
+  #  end
+  #
+  #  Foo.with_master do |db|
+  #    p db::SomeTable.where(:some_column = 'bar')
+  #  end
+  #  Foo.with_slave do |db|
+  #    p db::SomeTable.where(:some_column = 'bar')
+  #  end
+  #
+  #  Bar.with_master_for(id) do |db|
+  #    p db::SomeTable.where(:some_column = 'bar')
+  #  end
+  #  Bar.with_master_for_new do |db|
+  #    db::SomeTable.new do
+  #       ...
+  #    end
+  #  end
+  #  Bar.with_slave_for(id) do |db|
+  #    p db::SomeTable.where(:some_column = 'bar')
+  #  end
+  #
+  # See MyStuff::MultiDB::Unsharded and MyStuff::MultiDB::Sharded
+  #
+  # = Details
+  #
+  # When you call <tt>with_*</tt>, it:
+  # * Looks for a class called MyStuff::MultiDB::MANGLED_DATABASE_NAME
+  # * If it doesn't exist, it creates a new ActiveRecord::Base subclass
+  # * It then looks for a module within that class with the same name as your
+  #   module
+  # * If it's not there:
+  #   * It creates the module
+  #   * It creates a subclass of each of your ActiveRecord definitions within this
+  #     module
+  #   * It delegates connection handling to the database class
+  # * It then returns the database class
+  #
+  # So, you end up with an ActiveRecord class like:
+  # MyStuff::MultiDB::MANGLED_DATABASE_NAME::YourModule::YourClass
+  module MultiDB
+    def self.open_connections
+      Hash.new.tap do |result|
+        constants.each do |name|
+          if name.to_s.starts_with? 'MYSTUFF_MULTIDB_DB'
+            klass = const_get(name, false)
+            checked_out = klass.connection_pool.instance_eval { @checked_out.size }
+            result[name] = checked_out if checked_out > 0
+          end
+        end
+      end
+    end
+
+    # Takes a module name and converts it to connection details.
+    #
+    # Example:
+    # mangled::    <tt>MyStuff::MultiDB::MYSTUFF_MULTIDB_DB_747970686f6e2e66726564656d6d6f74742e636f2e756b2c333330362c747474::ServiceLocator::Tier</tt>
+    # unmanagled:: <tt>MyStuff::MultiDB::<mysql://typhon.fredemmott.co.uk:3306/ttt>::ServiceLocator::Tier</tt>
+    #
+    # The initscript cat-log and tail-log commands will do this unmanggling for you.
+    def self.unmangle name
+      db = name.sub /^:?MYSTUFF_MULTIDB_DB_/, ''
+      # Format: "MYSTUFF_MULTIDB_DB_" + hex("host,port,database")
+      junk, host, port, database = db.each_char.each_slice(2).reduce(String.new){ |m,*nibbles| m += "%c" % nibbles.join.hex }.split('!')
+      {
+        :host     => host,
+        :port     => port.to_i,
+        :database => database,
+      }
+    end
+
+    def self.included othermod # :nodoc:
+      class <<othermod
+        def for_spec spec
+          MyStuff::MultiDB.for_spec(spec, self)
+        end
+        def with_slave *args; raise NotImplementedError.new "Available in MyStuff::MultiDB::Unsharded"; end
+        def with_master *args; raise NotImplementedError.new "Available in MyStuff::MultiDB::Unsharded"; end
+        def with_master_for *args; raise NotImplementedError.new "Available in MyStuff::MultiDB::Sharded"; end
+        def with_master_for_new; raise NotImplementedError.new "Available in MyStuff::MultiDB::Sharded"; end
+        def with_slave_for *args; raise NotImplementedError.new "Available in MyStuff::MultiDB::Sharded"; end
+        def sharded?; raise NotImplementedError.new; end
+      end
+    end
+
+    # Fetch/create the magic classes.
+    def self.for_spec spec, mod # :nodoc:
+      db_key = ("MYSTUFF_MULTIDB_DB_" + ("!%s!%s!%s" % [
+          spec[:host],
+          spec[:port],
+          spec[:database],
+        ]
+      ).each_byte.map{ |x| "%x" % x }.join).to_sym
+
+      # db: class representing the logical database
+      if self.const_defined? db_key
+        db = self.const_get(db_key)
+      else
+        db = Class.new ActiveRecord::Base
+        def db.abstract_class?; true; end
+        self.const_set(db_key, db)
+        db.establish_connection(spec)
+      end
+
+      mod_key = mod.name.split(':').last.to_sym
+      # db_mod: class representing the module from the AR definition
+      if db.const_defined? mod_key, false
+        db_mod = db.const_get(mod_key, false)
+      else
+        db_mod = Module.new
+        db.const_set(mod_key, db_mod)
+        db_mod.send(:define_singleton_method, :magic_database) { db }
+        db_mod.send(:define_singleton_method, :muggle) { mod }
+
+        # klass: a specific table's AR class
+        def db_mod.const_missing name
+          klass = muggle.const_get(name)
+          klass_sym = klass.name.split(':').last.to_sym
+
+          # subklass: klass tied to a specific DB
+          subklass = Class.new(klass)
+          const_set klass_sym, subklass
+          define_singleton_method(klass_sym) { subklass }
+
+          subklass.send :include, MyStuff::MultiDB::Base
+
+          # Make associations work.
+          klass.reflect_on_all_associations.each do |reflection|
+            subklass.send(
+              reflection.macro, # eg :has_one
+              reflection.name, # eg :some_table
+              reflection.options
+            )
+          end
+
+          return subklass
+        end
+      end
+
+      return db_mod
+    end
+
+    protected
+    def self.with_db db, id, writable # :nodoc:
+      if writable == :writable
+        if id == :new
+          spec = db.spec_for_new
+        else
+          spec = db.spec_for_master(id)
+        end
+      elsif writable == :read_only
+        spec = db.spec_for_slave(id)
+      end
+      klass = db.for_spec(spec)
+
+      klass.magic_database.connection_pool.with_connection do
+        yield klass, spec
+      end
+    end
+  end
+end
+
+require 'my_stuff/multidb/base'
+require 'my_stuff/multidb/sharded'
+require 'my_stuff/multidb/unsharded'

data/lib/my_stuff/multidb/base.rb

@@ -0,0 +1,43 @@
+# Copyright 2011-present Fred Emmott. See COPYING file.
+
+module MyStuff
+  module MultiDB
+    module Base # :nodoc:
+      def connection
+        self.class.connection
+      end
+
+      def self.included(klass)
+        klass.extend(ClassMethods)
+      end
+     
+      module ClassMethods
+        def magic_database
+          @magic_database ||= self.name.split('::').tap(&:pop).join('::').constantize.magic_database
+        end
+
+        def arel_engine
+          magic_database.arel_engine
+        end
+
+        def connection
+          magic_database.connection
+        end
+
+        def abstract_class?; true; end
+
+        def model_name
+          # Rails form_for wants this
+          ActiveModel::Name.new(
+            self.name.split('::').last.tap{|s| def s.name; self; end}
+          )
+        end
+
+        def inherited(child)
+          def child.abstract_class?; false; end
+          super
+        end
+      end
+    end
+  end
+end

data/lib/my_stuff/multidb/sharded.rb

@@ -0,0 +1,29 @@
+# Copyright 2011-present Fred Emmott. See COPYING file.
+
+module MyStuff
+  module MultiDB
+    module Sharded
+      def self.included othermod # :nodoc:
+        othermod.send :include, MyStuff::MultiDB
+        class <<othermod
+          def sharded?; true; end;
+          def with_master_for id
+            MyStuff::MultiDB.with_db(
+              self, id, :writable
+            ) { |*args| yield *args}
+          end
+          def with_master_for_new
+            MyStuff::MultiDB.with_db(
+              self, :new, :read_only
+            ) { |*args| yield *args}
+          end
+          def with_slave_for id
+            MyStuff::MultiDB.with_db(
+              self, id, :read_only
+            ) { |*args| yield *args}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/my_stuff/multidb/unsharded.rb

@@ -0,0 +1,29 @@
+# Copyright 2011-present Fred Emmott. See COPYING file.
+
+module MyStuff
+  module MultiDB
+    # Mixin for databases with no sharding (eg ServiceLocator, which defines the sharding).
+    #
+    # See MyStuff::MultiDB for an example.
+    #
+    # Including this defines <tt>with_master</tt> and <tt>with_slave</tt>.
+    module Unsharded
+      def self.included othermod # :nodoc:
+        othermod.send :include, MyStuff::MultiDB
+        class <<othermod
+          def sharded?; false; end;
+          def with_master
+            MyStuff::MultiDB.with_db(
+              self, :unsharded, :writable
+            ) { |*args| yield *args }
+          end
+          def with_slave
+            MyStuff::MultiDB.with_db(
+              self, :unsharded, :read_only
+            ) { |*args| yield *args}
+          end
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification 
+name: my_stuff-multidb
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Fred Emmott
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-17 00:00:00 Z
+dependencies: []
+
+description: ""
+email: 
+- mail@fredemmott.co.uk
+executables: 
+- my_stuff-multidb-unmangle
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- COPYING
+- README.rdoc
+- bin/my_stuff-multidb-unmangle
+- lib/my_stuff/multidb/sharded.rb
+- lib/my_stuff/multidb/base.rb
+- lib/my_stuff/multidb/unsharded.rb
+- lib/my_stuff/multidb.rb
+homepage: https://github.com/fredemmott/my_stuff-multidb
+licenses: 
+- ISC
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: ActiveRecord sharding/slaves library
+test_files: []
+