ocelot 0.0.0

data/lib/ocelot.rb

@@ -0,0 +1,90 @@
+module Ocelot
+  class Base
+
+    def connection
+      Ocelot.connection
+    end
+  
+    def in?(*args)
+      Ocelot.in?(*args)
+    end
+
+    def remove_callbacks(*args)
+      Ocelot.remove_callbacks(*args)
+    end
+
+    def logger(*args)
+      Ocelot.logger(*args)
+    end
+
+    def obj_to_s(*args)
+      Ocelot.obj_to_s(*args)
+    end
+
+    def safely(*args, &b)
+      Ocelot.safely(*args) do |*bargs|
+        b.call(*bargs)
+      end
+    end
+  end
+
+  require 'ocelot/graph'
+  require 'ocelot/dsl'
+  require 'ocelot/rules'
+  require 'ocelot/database'
+  require 'ocelot/processor'
+
+  class << self
+    attr_accessor :connection
+    attr_accessor :logger
+
+    def obj_to_s(obj, assoc=nil)
+      assoc = assoc.nil? ? "" : ".#{assoc.name}"
+      "#{obj.class}(#{obj.id})#{assoc}" 
+    end
+
+    def safely(obj, assoc=nil, val=nil)
+      begin
+        yield
+      rescue => e
+        target = obj.is_a?(Class) ? "#{obj}" : "#{obj_to_s(obj)}"
+
+        if assoc.nil?
+          logger.error "Could not read or write #{target}: #{e.message}"
+        elsif val.nil?
+          logger.error "Could not read data for #{target}.#{assoc}: #{e.message}"
+        else
+          logger.error "Could not write #{target}.#{assoc} adding #{val.class}: #{e.message}"
+        end
+      end
+    end
+
+    # this should really be a class-id pair store instead of the entire object, 
+    # /maybe/
+    def in?(set, node)
+      result = set.include? node
+      set << node
+      result 
+    end
+
+    def remove_callbacks(obj)
+      if obj.class.respond_to? :delete_observers
+        obj.class.delete_observers
+      end
+
+      class << obj
+        def valid?
+          true
+        end
+      end
+
+      ActiveRecord::Callbacks::CALLBACKS.each do |cb| 
+        class << obj
+          def run_callbacks(*goaway)
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ocelot/cli.rb

@@ -0,0 +1,9 @@
+# Rails wants to include AR first.  
+puts "Loading Rails environment..."
+require File.join(ARGV[0], 'environment')
+
+require 'ocelot/dsl'
+
+go = Ocelot::Dsl.new
+go.instance_eval File.read(ARGV[1]), ARGV[1]
+go.go

data/lib/ocelot/database.rb

@@ -0,0 +1,41 @@
+require 'thread'
+require 'active_record'
+require 'active_record/connection_adapters/mysql2_adapter'
+require 'active_record/connection_adapters/abstract/connection_pool'
+
+module Ocelot
+  class Connection
+    attr_accessor :source
+    attr_accessor :target
+    attr_accessor :use_source
+
+    def initialize(source, target)
+      self.source = ActiveRecord::Base.mysql2_connection(source)
+      self.target = ActiveRecord::Base.mysql2_connection(target)
+      self.use_source = true
+
+      # ar doesn't like oop much, monkeypatch lots o stuff
+      ActiveRecord::ConnectionAdapters::ConnectionHandler.class_variable_set(:@@conn, self)
+      ActiveRecord::ConnectionAdapters::ConnectionHandler.class_eval do 
+        def retrieve_connection(klass)
+          @@conn.current_connection
+        end
+      end
+
+      ActiveRecord::ConnectionAdapters::AbstractAdapter.class_eval do
+        attr_reader :last_query
+
+        def log_with_last_query(sql, name, &block)
+          @last_query = [sql, name]
+          log_without_last_query(sql, name, &block)
+        end
+
+        alias_method_chain :log, :last_query
+      end
+    end
+
+    def current_connection
+      self.use_source ? source : target
+    end
+  end
+end

data/lib/ocelot/dsl.rb

@@ -0,0 +1,146 @@
+require 'ocelot'
+require 'ocelot/rules'
+
+module Ocelot
+  class Dsl
+    attr_accessor :rules
+    attr_accessor :filters
+
+    def initialize
+      @rules = {}
+      @filters = {}
+      @seeds = lambda { [] }
+      @logger = Logger.new(STDERR)
+    end
+
+    def source(source=nil)
+      @source = source unless source.nil?
+      @source
+    end
+
+    def seeds(&seeds)
+      @seeds = seeds unless seeds.nil?
+      @seeds
+    end
+   
+    def target(target=nil)
+      @target = target unless target.nil?
+      @target
+    end
+
+    def classes(classes=nil)
+      @classes = classes.split(/\s+/) unless classes.nil?
+      @classes
+    end
+
+    def logger
+      @logger
+    end
+
+    def log_level(level=nil)
+      @logger.level = eval "Logger::#{level}" unless level.nil?
+      level
+    end
+
+    def init(&p)
+      @init = p unless p.nil?
+      @init
+    end
+
+    def rule(clazz, &p) 
+      @rules[clazz] = ScriptRule.new(p)
+    end
+
+    def filter(clazz, &p)
+      @filters[clazz] = ScriptFilter.new(p)
+    end
+
+    def go
+      Ocelot::logger            = logger
+      Ocelot::connection        = Ocelot::Connection.new(source, target)
+
+      unless init.nil?
+        Ocelot.connection.use_source = false
+        init.call
+        Ocelot.connection.use_source = true
+      end
+
+      copier = Ocelot::Processor.new
+      copier.classes       = classes.collect { |c| eval c }
+      copier.seeds         = seeds.call.flatten
+      copier.extra_rules   = rules
+      copier.extra_filters = filters
+
+      if true
+        copier.go
+      else
+        copier_thread = Thread.new(copier) do
+          copier.go
+        end
+        copier_thread.run
+        copier_thread.set_trace_func proc { |*args| copier.watcher.trace(*args) }
+        last_warning = nil
+
+        # pole status
+        while !copier_thread.status.nil? and copier_thread.status != false
+          copier_thread.join(30)
+          warning = copier.watcher.ping
+          unless warning.nil? or last_warning == warning
+            logger.warn warning 
+            last_warning = warning
+          end
+        end
+      end
+    end
+  end
+
+  class ScriptFilter < Ocelot::Rules::Filter
+    alias :old_load :load
+    def initialize(p)
+      @p = p
+    end
+
+    def load(obj, collection, name)
+      @obj, @collection, @name = obj, collection, name
+
+      @result = nil
+      instance_exec(obj, collection, name, &@p)
+      @result || default
+    end
+
+    def nothing
+      @result = []
+    end
+
+    def result(val)
+      @result = val
+    end
+
+    def default
+      @result = old_load @obj, @collection, @name 
+    end
+  end
+
+  class ScriptRule < Ocelot::Rules::DefaultRule
+    alias :old_save :save
+
+    def initialize(p)
+      @p = p
+    end
+
+    def inspect
+      @inspect = true 
+    end
+
+    def persist(obj)
+      old_save obj 
+    end
+
+    def save(obj, *args)
+      @inspect = false
+      args = [obj, *args]
+      instance_exec(*args, &@p)
+      return @inspect
+    end
+  end
+end

data/lib/ocelot/graph.rb

@@ -0,0 +1,14 @@
+module Ocelot
+  module Graph
+    class Traverser
+      def self.run(queue)
+        while !queue.empty?
+          snapshot, queue = queue.uniq, []
+          snapshot.each do |node|
+            yield node, queue
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ocelot/processor.rb

@@ -0,0 +1,182 @@
+module Ocelot
+  class Watcher
+    attr_accessor :op
+    attr_accessor :association
+    attr_accessor :sql
+    attr_accessor :line
+    attr_accessor :rule
+    attr_reader   :last
+    attr_reader   :obj
+
+    def initialize
+      @last = Time.now
+      @mutex = Mutex.new
+    end
+
+    def obj=(obj)
+      @obj = Ocelot::obj_to_s(obj)
+    end
+
+    def ping
+      sync do |w|
+        "Might be stuck processing #{w.obj}#{w.association.nil? ? "" : "."}#{w.association} at #{w.line} using #{w.rule}, may be exec'ing #{ActiveRecord::Base.connection.last_query}" if Time.now - w.last > 20
+      end
+    end
+
+    def sync
+      @mutex.synchronize do
+        yield self
+      end
+    end
+ 
+    def trace(event, file, line, id, binding, clazz)
+      if clazz == Ocelot::Processor
+        @last = Time.now
+        @line = line
+      end    
+    end
+  end
+
+  class Processor < Base
+    attr_accessor :seeds
+    attr_accessor :classes
+    attr_accessor :extra_rules
+    attr_accessor :extra_filters
+    attr_reader   :watcher
+    attr_reader   :saved
+
+    def initialize
+      @extra_rules = {}
+      @extra_filters = {}
+      @seeds = []
+      @unrecognised = Set.new
+      @watcher = Watcher.new
+    end
+
+    def process_filter(params)
+      obj        = params.delete :obj
+      collection = params.delete :collection 
+      name       = params.delete :name
+
+      result = nil
+      filter = @filters[obj.class]
+
+      if !filter.nil?
+        watcher.sync do |w|
+          w.obj = obj
+          w.association = name
+          w.rule = filter
+        end
+
+        logger.info "Filtering #{obj_to_s(obj)}.#{name} via #{filter.class}"
+        result = filter.load(obj, collection, name)
+      elsif !@unrecognised.include? obj.class
+        logger.error "Unrecognised type #{obj.class}"
+        @unrecognised << obj.class
+      end
+      result || []
+    end
+
+    def process_rule(params)
+      obj     = params.delete :obj
+      src     = params.delete :src
+      target  = params.delete :target
+      calller = params.delete :caller 
+
+      rule = @rules[obj.class]
+
+      watcher.sync do |w|
+        w.obj = obj
+        w.association = nil
+        w.rule = rule
+      end
+
+      if !rule.nil?
+        logger.info "Saving #{obj_to_s(obj)} thru #{rule.class} via #{calller}"
+        return rule.save(obj, src, target)
+      elsif !@unrecognised.include? obj.class
+        logger.warn "Unrecognised type #{obj.class}"
+        @unrecognised << obj.class
+      end
+    end
+
+    def go
+      default_rule   = Rules::DefaultRule.new
+      default_filter = Rules::Filter.new
+
+      @rules     = Hash.new
+      @filters   = Hash.new
+      @saved     = Set.new   
+      @inspected = Set.new   
+
+      array_rule   = ArrayRule.new
+ 
+      @classes.each { |c| @rules[c] = default_rule; @filters[c] = default_filter }
+      @rules.merge! @extra_rules
+      @filters.merge! @extra_filters
+      seeds = @seeds.collect { |o| [o, "script seed"] } 
+
+      Graph::Traverser.run(seeds) do |objs, queue|
+        obj, calller = *objs
+
+        unless in? @saved, obj
+          @inspected << obj unless process_rule(:obj => obj, :caller => calller)
+        end
+
+        unless in? @inspected, obj
+          obj.class.reflect_on_all_associations.each do |assoc| 
+            safely obj do
+              connection.use_source = true
+              value = obj.send(assoc.name)
+
+              unless value.nil?
+ 
+                # simple has_one (A->B handling), save B and conditionally inspect
+                if !assoc.collection?
+                  unless in? @saved, value
+                    process_rule(:obj => value, :caller => calller) and (queue << [value, obj_to_s(obj, assoc)])
+                  end
+                # more complicated has many (A-B* and A-mapobj-B*), saving and inspecting each child
+                # monkeypatching the array_rule must yield B and whether to inspect it
+                # be very very very careful, as it handles saving objects whie yielding results
+                else
+                  array_rule.save(obj, value, assoc, self) do |child, inspect| 
+                    @saved << child
+                    if inspect
+                      queue << [child, obj_to_s(obj, assoc)] 
+                    else
+                      @inspected << child 
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+
+    class ArrayRule < Rules::Rule
+      include Ocelot
+
+      def save(obj, array, assoc, c)
+        safely obj, assoc.name do
+          c.process_filter(:obj => obj, :collection => array, :name => assoc.name).each do |val| 
+            unless in? c.saved, val 
+              use_source_db!
+              inspect = c.process_rule(:obj => val, :caller => obj_to_s(obj)) 
+              yield val, inspect
+            end
+
+            use_target_db!
+            if assoc.is_a? ActiveRecord::Reflection::ThroughReflection
+              safely assoc.through_reflection.klass, assoc.name, val do
+                c.process_rule :obj => assoc.through_reflection.klass.new, :src => obj, :target => val, :caller => obj_to_s(obj,assoc)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ocelot/rules.rb

@@ -0,0 +1,53 @@
+module Ocelot
+  module Rules
+    class Filter < Ocelot::Base
+      def filter(value)
+        true
+      end
+
+      def load(obj, collection, name)
+        result = []
+        collection.each do |val|
+          safely(obj, name) do
+            result << val if filter val
+          end
+        end
+      end
+    end
+
+
+    class Rule < Base
+      def relationships(obj)
+        obj.class.reflect_on_all_associations.collect { |assoc| assoc.name }
+      end
+
+      def use_target_db!
+        Ocelot::connection.use_source = false
+      end
+
+      def use_source_db!
+        Ocelot::connection.use_source = true
+      end
+
+      def save_to_db(obj, *extra)
+        begin
+          use_target_db!
+          obj.save(false)
+        ensure
+          use_source_db!
+        end
+      end
+    end  
+
+    class DefaultRule < Rule
+      def save(obj, *extra)
+        id = obj.id
+        obj = obj.clone 
+        obj.id = id
+        remove_callbacks(obj)
+        save_to_db(obj)
+        true
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification 
+name: ocelot
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.0
+platform: ruby
+authors: 
+- spencer p
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-04-17 00:00:00 -04:00
+default_executable: 
+dependencies: []
+
+description: Extreme alpha
+email: 
+- exussum@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/ocelot/cli.rb
+- lib/ocelot/database.rb
+- lib/ocelot/dsl.rb
+- lib/ocelot/graph.rb
+- lib/ocelot/processor.rb
+- lib/ocelot/rules.rb
+- lib/ocelot.rb
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: DBA like tool to copy a slice of data from one db to another
+test_files: []
+