my_stuff-multidb 0.0.3 → 0.1.0

data/COPYING

@@ -1,4 +1,4 @@
-Copyright (c) 2011-present, Fred Emmott <copyright@fredemmott.co.uk>
+Copyright (c) 2011-2012, Fred Emmott <copyright@fredemmott.co.uk>
 
 Permission to use, copy, modify, and/or distribute this software for any
 purpose with or without fee is hereby granted, provided that the above

data/bin/my_stuff-multidb-unmangle

@@ -5,11 +5,11 @@ _FILE = File.readlink(__FILE__) rescue __FILE__
 _DIR = File.expand_path(File.dirname(_FILE))
 $LOAD_PATH.push(File.expand_path(_DIR +  '/../lib'))
 
-require 'my_stuff/multidb'
+require 'my_stuff/multidb/mangling'
 
 def filter line
   line.gsub(/MYSTUFF_MULTIDB_DB_[a-z0-9]+/){ |mangled|
-    data = MyStuff::MultiDB.unmangle(mangled)
+    data = MyStuff::MultiDB::Mangling.unmangle(mangled)
     "<%s:%d/%s>" % [
       data[:host],
       data[:port],

data/lib/my_stuff/multidb.rb

@@ -1,7 +1,12 @@
 # Copyright 2011-present Fred Emmott. See COPYING file.
 
+require 'my_stuff/multidb/connection'
+require 'my_stuff/multidb/core_ext/base'
+
 require 'base64'
-require 'my_stuff/logger'
+
+require 'rubygems'
+require 'active_record'
 
 module MyStuff
   # =Example
@@ -55,135 +60,28 @@ module MyStuff
   # So, you end up with an ActiveRecord class like:
   # MyStuff::MultiDB::MANGLED_DATABASE_NAME::YourModule::YourClass
   module MultiDB
-    def self.open_connections
-      Hash.new.tap do |result|
-        constants.each do |name|
-          if name.to_s.starts_with? 'MYSTUFF_MULTIDB_DB'
-            klass = const_get(name, false)
-            checked_out = klass.connection_pool.instance_eval { @checked_out.size }
-            result[name] = checked_out if checked_out > 0
-          end
-        end
-      end
-    end
-
-    # Takes a module name and converts it to connection details.
-    #
-    # Example:
-    # mangled::    <tt>MyStuff::MultiDB::MYSTUFF_MULTIDB_DB_747970686f6e2e66726564656d6d6f74742e636f2e756b2c333330362c747474::ServiceLocator::Tier</tt>
-    # unmanagled:: <tt>MyStuff::MultiDB::<mysql://typhon.fredemmott.co.uk:3306/ttt>::ServiceLocator::Tier</tt>
-    #
-    # The initscript cat-log and tail-log commands will do this unmanggling for you.
-    def self.unmangle name
-      db = name.sub /^:?MYSTUFF_MULTIDB_DB_/, ''
-      # Format: "MYSTUFF_MULTIDB_DB_" + hex("host,port,database")
-      junk, host, port, database = db.each_char.each_slice(2).reduce(String.new){ |m,*nibbles| m += "%c" % nibbles.join.hex }.split('!')
-      {
-        :host     => host,
-        :port     => port.to_i,
-        :database => database,
-      }
-    end
-
     def self.included othermod # :nodoc:
       class <<othermod
-        def for_spec spec
-          MyStuff::MultiDB.for_spec(spec, self)
-        end
-        def with_slave *args; raise NotImplementedError.new "Available in MyStuff::MultiDB::Unsharded"; end
-        def with_master *args; raise NotImplementedError.new "Available in MyStuff::MultiDB::Unsharded"; end
-        def with_master_for *args; raise NotImplementedError.new "Available in MyStuff::MultiDB::Sharded"; end
-        def with_master_for_new; raise NotImplementedError.new "Available in MyStuff::MultiDB::Sharded"; end
-        def with_slave_for *args; raise NotImplementedError.new "Available in MyStuff::MultiDB::Sharded"; end
-        def sharded?; raise NotImplementedError.new; end
-      end
-    end
-
-    # Fetch/create the magic classes.
-    def self.for_spec spec, mod # :nodoc:
-      db_key = ("MYSTUFF_MULTIDB_DB_" + ("!%s!%s!%s" % [
-          spec[:host],
-          spec[:port],
-          spec[:database],
-        ]
-      ).each_byte.map{ |x| "%x" % x }.join).to_sym
-
-      # db: class representing the logical database
-      if self.const_defined? db_key
-        db = self.const_get(db_key)
-      else
-        l 'connecting to database: ', spec
-        db = Class.new ActiveRecord::Base
-        def db.abstract_class?; true; end
-        self.const_set(db_key, db)
-        db.establish_connection(spec)
-      end
-
-      mod_key = mod.name.split(':').last.to_sym
-      # db_mod: class representing the module from the AR definition
-      if db.const_defined? mod_key, false
-        db_mod = db.const_get(mod_key, false)
-      else
-        db_mod = Module.new
-        db.const_set(mod_key, db_mod)
-        db_mod.send(:define_singleton_method, :magic_database) { db }
-        db_mod.send(:define_singleton_method, :muggle) { mod }
-
-        # klass: a specific table's AR class
-        def db_mod.const_missing name
-          klass = muggle.const_get(name)
-          klass_sym = klass.name.split(':').last.to_sym
-
-          # subklass: klass tied to a specific DB
-          subklass = Class.new(klass)
-          const_set klass_sym, subklass
-          define_singleton_method(klass_sym) { subklass }
-
-          subklass.send :include, MyStuff::MultiDB::Base
-
-          # Make associations work.
-          klass.reflect_on_all_associations.each do |reflection|
-            subklass.send(
-              reflection.macro, # eg :has_one
-              reflection.name, # eg :some_table
-              reflection.options
-            )
-          end
-
-          return subklass
+        def with_spec spec, &block
+          MyStuff::MultiDB.with_spec(self, spec, &block)
         end
       end
-
-      return db_mod
     end
 
-    def self.with_spec db, spec # :nodoc:
-      klass = db.for_spec(spec)
-
-      klass.magic_database.connection_pool.with_connection do
-        yield klass, spec
-      end
-    end
+    def self.with_spec original_module, spec, &block # :nodoc:
+      ar_base = Connection.base_class_for_spec(spec)
+      rebased_module = ar_base.rebased_module(original_module)
 
-    protected
-    def self.with_db db, id, writable # :nodoc:
-      if writable == :writable
-        if id == :new
-          spec = db.spec_for_new
+      ar_base.connection_pool.with_connection do
+        if block.arity == 1
+          block.call rebased_module
         else
-          spec = db.spec_for_master(id)
+          block.call rebased_module, spec
         end
-      elsif writable == :read_only
-        spec = db.spec_for_slave(id)
-      end
-
-      with_spec(db, spec) do |*args|
-        yield *args
       end
     end
   end
 end
 
-require 'my_stuff/multidb/base'
 require 'my_stuff/multidb/sharded'
 require 'my_stuff/multidb/unsharded'

data/lib/my_stuff/multidb/connection.rb

@@ -0,0 +1,108 @@
+# Copyright 2011-present Fred Emmott. See COPYING file.
+
+require 'my_stuff/multidb/mangling'
+require 'my_stuff/multidb/core_ext/base'
+
+require 'rubygems'
+require 'active_record'
+
+module MyStuff
+  module MultiDB
+    module Connections; end
+
+    class Connection < ActiveRecord::Base
+      class << self
+        def base_class_for_spec spec
+          name = MyStuff::MultiDB::Mangling.mangle(spec).to_sym
+          if Connections.const_defined?(name)
+            return Connections.const_get(name)
+          end
+
+          connection = Class.new(self)
+          Connections.const_set(name, connection)
+          connection.establish_connection(spec)
+          connection
+        end
+
+        def abstract_class?
+          true
+        end
+
+        def rebased_module original_module
+          name = original_module.name.gsub(':', '__').to_sym
+
+          if have_rebased_module?(name)
+            self.const_get(name)
+          else
+            self.rebase_module! name, original_module
+          end
+        end
+
+        def rebased_model name, original_module, rebased_module
+          if rebased_module.const_defined? name
+            rebased_module.const_get(name)
+          else
+            self.rebase_model! name, original_module, rebased_module
+          end
+        end
+
+        protected
+
+        def rebase_model! name, original_module, rebased_module
+          name = name.to_sym
+          original = original_module.const_get(name)
+
+          rebased = Class.new(original)
+          rebased_module.const_set(name, rebased)
+          rebased.send :include, MyStuff::MultiDB::CoreExt::Base
+
+          # Make associations work.
+          rebased.reflect_on_all_associations.each do |reflection|
+            rebased.send(
+              reflection.macro, # eg :has_one
+              reflection.name, # eg :some_table
+              reflection.options
+            )
+          end
+
+          return rebased
+        end
+
+        def rebase_module! name, original_module
+          ar_base = self
+          rebased = Module.new
+          ar_base.const_set(name, rebased)
+
+          # Generate wrapper classes on demand
+          def rebased.const_missing (name)
+            MyStuff::MultiDB::Connection.rebased_model(
+              name,
+              muggle,
+              magic_database
+            )
+          end
+
+          # Not using define_singleton_method, as that's not in 1.8.7
+          singleton = class << rebased; self; end
+          singleton.send(:define_method, :magic_database) { ar_base }
+          singleton.send(:define_method, :muggle) { original_module }
+
+          return rebased
+        end
+
+        def have_rebased_module? name
+          # 1.8.7: const_defined? does not include constants defined
+          #   in other modules, and it only takes 1 arg
+          # 1.9: it does include, and needs a second argument to change
+          #   this.
+          old_const_defined = self.method(:const_defined?).arity == 1
+          new_const_defined = !old_const_defined
+          return (
+            (old_const_defined && self.const_defined?(name)) ||
+            (new_const_defined && self.const_defined?(name, false))
+          )
+        end
+      end
+    end
+  end
+end

data/lib/my_stuff/multidb/core_ext/base.rb

@@ -0,0 +1,59 @@
+# Copyright 2011-present Fred Emmott. See COPYING file.
+
+module MyStuff
+  module MultiDB
+    module CoreExt
+      # Mixin for subclasses of ActiveRecord::Base.
+      #
+      # This deals with routing connections via the 'magic'
+      # ActiveRecord::Base subclass for the appropriate database.
+      module Base # :nodoc:
+        def connection
+          self.class.connection
+        end
+
+        def self.included(klass)
+          klass.extend(ClassMethods)
+        end
+
+        module ClassMethods
+          def base_class
+            self
+          end
+
+          def magic_database
+            @magic_database ||=
+              self.name.split('::')[0..-2].join('::').constantize
+          end
+
+          def arel_engine
+            magic_database.arel_engine
+          end
+
+          def connection
+            magic_database.connection
+          end
+
+          def connection_pool
+            magic_database.connection_pool
+          end
+
+          def abstract_class?; true; end
+
+          def model_name
+            # Rails form_for wants this
+            ActiveModel::Name.new(
+              self.name.split('::').last.tap{|s| def s.name; self; end}
+            )
+          end
+
+          def inherited(child)
+            def child.abstract_class?; false; end
+            def child.base_class; self; end
+            super
+          end
+        end
+      end
+    end
+  end
+end

data/lib/my_stuff/multidb/mangling.rb

@@ -0,0 +1,38 @@
+# Copyright 2011-2012 Fred Emmott. See COPYING file.
+
+module MyStuff
+  module MultiDB
+    module Mangling
+      # Takes a module name and converts it to connection details.
+      #
+      # Example:
+      # mangled::    <tt>MyStuff::MultiDB::MYSTUFF_MULTIDB_DB_747970686f6e2e66726564656d6d6f74742e636f2e756b2c333330362c747474::ServiceLocator::Tier</tt>
+      # unmanagled:: <tt>MyStuff::MultiDB::<mysql://typhon.fredemmott.co.uk:3306/ttt>::ServiceLocator::Tier</tt>
+      #
+      # The initscript cat-log and tail-log commands will do this unmanggling for you.
+      def self.unmangle name
+        db = name.sub /^:?MYSTUFF_MULTIDB_DB_/, ''
+        # Format: "MYSTUFF_MULTIDB_DB_" + hex("host,port,database")
+        host, port, database = db.each_char.each_slice(2).reduce(String.new){ |m,*nibbles| m += "%c" % nibbles.join.hex }.split('!')
+        {
+          :host     => host,
+          :port     => port.to_i,
+          :database => database,
+        }
+      end
+
+      # Create a valid constant name from an ActiveRecord spec.
+      #
+      # This can be revered by {#unmangle}.
+      def self.mangle spec
+        'MYSTUFF_MULTIDB_DB_' + (
+          "%s!%s!%s" % [
+            spec[:host] || spec['host'],
+            spec[:port] || spec['port'],
+            spec[:database] || spec['database'],
+          ]
+        ).each_byte.map{ |x| "%x" % x }.join
+      end
+    end
+  end
+end

data/lib/my_stuff/multidb/sharded.rb

@@ -1,27 +1,41 @@
 # Copyright 2011-present Fred Emmott. See COPYING file.
 
+require 'my_stuff/multidb'
+
 module MyStuff
   module MultiDB
     module Sharded
       def self.included othermod # :nodoc:
         othermod.send :include, MyStuff::MultiDB
+
         class <<othermod
-          def sharded?; true; end;
-          def with_master_for id
-            MyStuff::MultiDB.with_db(
-              self, id, :writable
-            ) { |*args| yield *args}
+          def with_master_for id, &block
+            MyStuff::MultiDB.with_spec(
+              self,
+              self.spec_for_master(id),
+              &block
+            )
           end
-          def with_master_for_new
-            MyStuff::MultiDB.with_db(
-              self, :new, :read_only
-            ) { |*args| yield *args}
+
+          def with_master_for_new &block
+            MyStuff::MultiDB.with_spec(
+              self,
+              self.spec_for_new,
+              &block
+            )
           end
-          def with_slave_for id
-            MyStuff::MultiDB.with_db(
-              self, id, :read_only
-            ) { |*args| yield *args}
+
+          def with_slave_for id, &block
+            MyStuff::MultiDB.with_spec(
+              self,
+              self.spec_for_slave(id),
+              &block
+            )
           end
+
+          def spec_for_new; raise NotImplementedError.new; end
+          def spec_for_master(shard_id); raise NotImplementedError.new; end
+          def spec_for_slave(shard_id); raise NotImplementedError.new; end
         end
       end
     end

data/lib/my_stuff/multidb/unsharded.rb

@@ -1,4 +1,6 @@
 # Copyright 2011-present Fred Emmott. See COPYING file.
+#
+require 'my_stuff/multidb'
 
 module MyStuff
   module MultiDB
@@ -10,18 +12,26 @@ module MyStuff
     module Unsharded
       def self.included othermod # :nodoc:
         othermod.send :include, MyStuff::MultiDB
+
         class <<othermod
-          def sharded?; false; end;
-          def with_master
-            MyStuff::MultiDB.with_db(
-              self, :unsharded, :writable
-            ) { |*args| yield *args }
+          def with_master &block
+            MyStuff::MultiDB.with_spec(
+              self,
+              self.spec_for_master,
+              &block
+            )
           end
-          def with_slave
-            MyStuff::MultiDB.with_db(
-              self, :unsharded, :read_only
-            ) { |*args| yield *args}
+
+          def with_slave &block
+            MyStuff::MultiDB.with_spec(
+              self,
+              self.spec_for_slave,
+              &block
+            )
           end
+
+          def spec_for_master; raise NotImplementedError.new; end
+          def spec_for_slave; raise NotImplementedError.new; end
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: my_stuff-multidb
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,30 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-09-04 00:00:00.000000000Z
+date: 2012-02-13 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: my_stuff-logger
-  requirement: &70289582101220 !ruby/object:Gem::Requirement
+  name: activerecord
+  requirement: &70321517374640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.3
-  type: :runtime
-  prerelease: false
-  version_requirements: *70289582101220
-- !ruby/object:Gem::Dependency
-  name: activerecord
-  requirement: &70289582100640 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
+        version: '3.2'
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.2.1
   type: :runtime
   prerelease: false
-  version_requirements: *70289582100640
+  version_requirements: *70321517374640
 description: ''
 email:
 - mail@fredemmott.co.uk
@@ -42,9 +34,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - COPYING
-- README.rdoc
 - bin/my_stuff-multidb-unmangle
-- lib/my_stuff/multidb/base.rb
+- lib/my_stuff/multidb/connection.rb
+- lib/my_stuff/multidb/core_ext/base.rb
+- lib/my_stuff/multidb/mangling.rb
 - lib/my_stuff/multidb/sharded.rb
 - lib/my_stuff/multidb/unsharded.rb
 - lib/my_stuff/multidb.rb

data/README.rdoc

@@ -1,65 +0,0 @@
-= Overview
-
-This provides an API that's relatively convenient for:
-* Reading from slaves
-* Using multiple shards
-
-= Usage
-
- module MySpecProvider
-   def spec_for_new
-     { :adapter => 'mysql', :host => ...}
-   end
-
-   def spec_for_master shard_id
-     ...
-   end
-
-   def spec_for_slave shard_id
-     ...
-   end
- end
-
- module MyDB
-   class Widget < ActiveRecord::Base; end
-   include MyStuff::MultiDB::Sharded
-   extend MySpecProvider
- end
-
- MyDB.with_master_for_new do |db,spec|
-   db::Widget.create(...)
- end
-
- MyDB.with_slave_for(shard_id) do |db|
-   db::Widget.find(record_id);
-   ...
- end
-
-Another option, if you only do primary key lookups, is to encode the shard
-ID into the record id.
-
-For more, see:
-
-* ./examples/run
-* comments at top of lib/mystuff/multidb.rb
-
-= How It Works
-
-For every spec, it defines a new sub-module of MyStuff::MultiDB, which
-encodes the database details, and creates new subclasses of your
-ActiveRecord::Base classes, within these modules. For example:
-
- $ bin/my_stuff-multidb-unmangle MyStuff::MultiDB::MYSTUFF_MULTIDB_DB_216c6f63616c686f7374213333303621::MyDB::Widget
- MyStuff::MultiDB::<localhost:3306/>::MyDB::Widget
- $
-
-There's also lots of deep voodoo making the ActiveRecord stack use the right
-connection details :)
-
-= Caveats
-
-You can not do cross-shard operations simply, eg joins.
-
-= Copying
-
-See the COPYING file.

data/lib/my_stuff/multidb/base.rb

@@ -1,58 +0,0 @@
-# Copyright 2011-present Fred Emmott. See COPYING file.
-
-module MyStuff
-  module MultiDB
-    module Base # :nodoc:
-      def connection
-        self.class.connection
-      end
-
-      def self.included(klass)
-        klass.extend(ClassMethods)
-      end
-     
-      module ClassMethods
-        def base_class
-          self
-        end
-
-        def with_spec spec
-          MyStuff::MultiDB.with_spec(
-            db, spec
-          ) { |*args| yield *args }
-        end
-
-        def magic_database
-          @magic_database ||= self.name.split('::').tap(&:pop).join('::').constantize.magic_database
-        end
-
-        def arel_engine
-          magic_database.arel_engine
-        end
-
-        def connection
-          magic_database.connection
-        end
-
-        def connection_pool
-          magic_database.connection_pool
-        end
-
-        def abstract_class?; true; end
-
-        def model_name
-          # Rails form_for wants this
-          ActiveModel::Name.new(
-            self.name.split('::').last.tap{|s| def s.name; self; end}
-          )
-        end
-
-        def inherited(child)
-          def child.abstract_class?; false; end
-          def child.base_class; self; end
-          super
-        end
-      end
-    end
-  end
-end