transfer 0.0.1

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in transfer.gemspec
+gemspec

data/README.md

@@ -0,0 +1,172 @@
+## Transfer Gem
+Transfer data from source database to ActiveRecord, SequelModel or Mongoid models.
+
+### Installation
+Include Transfer in Gemfile:
+
+```ruby
+gem 'transfer'
+```
+
+You can run bundle from command line:
+
+```console
+bundle install
+```
+
+
+### Compatibility
+Source database: all, supported by [Sequel](http://sequel.rubyforge.org/documentation.html).
+
+Destination: ActiveRecord, SequelModel, Mongoid.
+
+
+## Configure
+Set connection options of source database and global parameters:
+
+```ruby
+Transfer.configure do |c|
+  c.host = "localhost"
+  c.adapter = "postgres"
+  c.database = "source_database"
+  c.user = "username"
+  c.password = "password"
+end
+```
+
+Available options:
+
+* `validate` on/off model validations. Values: `true` or `false`, default is `false`.
+* `failure_strategy` sets strategy if save of model is not successfully. Values: `:ignore` or `:rollback`, defult is `:ignore`.
+* `before` [global callback](#global_callbacks).
+* `success` [global callback](#global_callbacks).
+* `failure` [global callback](#global_callbacks).
+* `after` [global callback](#global_callbacks).
+* another options interpreted as [Sequel database connection options](http://sequel.rubyforge.org/rdoc/files/doc/opening_databases_rdoc.html).
+
+
+## Usage
+Direct transfer from source table `:users` to `User` model. All columns, including protected, existing in source table and destination model, will transferred:
+
+```ruby
+transfer :users => User
+```
+
+Filling the field `country` a constant value:
+
+```ruby
+transfer :users => User do
+  country "England"
+end
+```
+Transfer `:name` column from source table `:users` into `first_name` of `User` model:
+
+```ruby
+transfer :users => User do
+  first_name :name
+end
+```
+To produce, dynamic value (e.g. `dist_name` field), you can pass a block and access the row of source table:
+
+```ruby
+transfer :users => User do
+  dist_name {|row| "Mr. #{row[:first_name]}"}
+end
+```
+
+### Global callbacks <a name="global_callbacks"/>
+This callbacks called for each `transfer`.
+
+```ruby
+Transfer.configure do |c|
+  c.before do |klass, dataset|
+    #...
+  end
+  c.success do |model, row|
+    #...
+  end
+  c.failure do |model, row, exception|
+    #...
+  end
+  c.after do |klass, dataset|
+    #...
+  end
+end
+```
+Available global callbacks:
+
+* `before` called before an transfer started. Parameters: `klass`, `dataset`.
+* `success` called if save model is successfully. Parameters: `model`, `row`.
+* `failure` called if save model is not successfully. Parameters: `row`, `exception`.
+* `after` called after an transfer finished. Parameters: `klass`, `dataset`.
+
+Description of parameters:
+
+* `dataset` source table dataset, instance of [Sequel::Dataset](http://sequel.rubyforge.org/rdoc/classes/Sequel/Dataset.html).
+* `klass` is destination class.
+* `model` builded model, instance of `klass`.
+* `row` of source table. Type: `Hash`.
+* `exception` if save of model is not successfull.
+
+
+### Local transfer callbacks
+You can specify callbacks in your `transfer` that are separate from the model callbacks. This callbacks called in model context, therefore `self` keyword points to model.
+
+```ruby
+transfer :users => User do
+  before_save do |row|
+    self.messages << Message.build(:title => "Transfer", :description => "Welcome to new site, #{row[:fname]}!")
+  end
+end
+```
+Available callbacks:
+
+* `before_save` called before save model. Paramaters: `row`.
+* `after_save` called after successfully save model. Parameters: `row`.
+
+where `row` is row of source table, type: `Hash`.
+
+
+### Filter columns
+`only` filter passes source columns, specified in parameters.
+
+```ruby
+transfer :users => User do
+  only :name
+end
+```
+`except` filter passes all source columns, except for those that are specified in the parameters:
+
+```ruby
+transfer :users => User do
+  except :name
+end
+```
+
+
+### Replace global options
+Global options can be replaced global options, if it passed to `transfer`.
+
+```ruby
+transfer :users => User, :validate => false, :failure_strategy => :rollback
+```
+Available options for replace:
+
+* `validate`
+* `failure_strategy`
+
+
+### Logging
+If you also want see progress of transfer in console, use e.g. [progressbar gem](https://github.com/peleteiro/progressbar) with global callbacks.
+
+```ruby
+require 'progressbar'
+
+Transfer.configure do |c|
+  c.before {|klass, dataset| @pbar = ProgressBar.new(klass, dataset.count) }
+  c.success { @pbar.inc }
+  c.after { @pbar.halt }
+end
+
+transfer :users => User
+```

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/transfer/config.rb

@@ -0,0 +1,34 @@
+require 'sequel'
+
+module Transfer
+  class Config
+    def initialize data = {}
+      data.each do |key, value|
+        send "#{key}=", value
+      end
+    end
+
+    def connection_options
+      @connection_options ||= {}
+    end
+
+    def process_options
+      @process_options ||= { :validate => true, :failure_strategy => :ignore }
+    end
+
+    def connection
+      @connection ||= Sequel.connect connection_options
+    end
+
+    def method_missing name, *args, &block
+      /^((validate|failure_strategy|failure|before|after|success)|\w+)(=)?$/.match name
+      opt = $2 ? process_options : connection_options
+      if block_given?
+        opt[$1.to_sym] = block
+      else
+        $3 ? opt[$1.to_sym] = args[0] : opt[name]
+      end
+    end
+  end
+
+end

data/lib/transfer/generators/active_record.rb

@@ -0,0 +1,31 @@
+class Transfer::Generators::ActiveRecord < Transfer::Generators::Base
+
+  def self.supports? klass
+    defined?(ActiveRecord) && klass.ancestors.include?(ActiveRecord::Base)
+  end
+
+  def column_present? name
+    super(name) || klass.column_names.include?(name.to_s)
+  end
+
+  def transaction &block
+    klass.transaction &block
+  end
+
+  def create attributes, row, options={}, callbacks={}
+    model = klass.new attributes, :without_protection => true
+
+    model.instance_exec(row, &callbacks[:before_save]) if callbacks[:before_save]
+    save_options = options.select{|key| key == :validate }
+
+    model.save! save_options
+    options[:success].call(model, row) if options[:success]
+    model.instance_exec(row, &callbacks[:after_save]) if callbacks[:after_save]
+    model
+  rescue Exception => e
+    options[:failure].call(model, row, e) if options[:failure]
+    raise ActiveRecord::Rollback if options[:failure_strategy] == :rollback
+    model
+  end
+
+end

data/lib/transfer/generators/base.rb

@@ -0,0 +1,26 @@
+class Transfer::Generators::Base
+  attr_accessor :klass
+
+  def self.supports? klass
+    raise "Not Yet Implemented!"
+  end
+
+  def initialize klass
+    @klass = klass
+  end
+
+  def before
+  end
+
+  def after
+  end
+
+  def create attributes, row, options={}, callbacks={}
+    raise "Not Yet Implemented!"
+  end
+
+  def column_present? name
+    klass.method_defined? name
+  end
+
+end

data/lib/transfer/generators/mongoid.rb

@@ -0,0 +1,28 @@
+class Transfer::Generators::Mongoid < Transfer::Generators::Base
+
+  def self.supports? klass
+    defined?(Mongoid) && klass.ancestors.include?(Mongoid::Document)
+  end
+
+  def transaction
+    yield
+  rescue
+    klass.delete_all
+  end
+
+  def create attributes, row, options={}, callbacks={}
+    model = klass.new attributes, :without_protection => true
+    model.instance_exec(row, &callbacks[:before_save]) if callbacks[:before_save]
+    save_options = options.select{|key| key == :validate }
+    model.save! save_options
+
+    options[:success].call(model, row) if options[:success]
+    model.instance_exec(row, &callbacks[:after_save]) if callbacks[:after_save]
+    model
+  rescue Exception => e
+    options[:failure].call(model, row, e) if options[:failure]
+    raise if options[:failure_strategy] == :rollback
+    model
+  end
+
+end

data/lib/transfer/generators/sequel.rb

@@ -0,0 +1,47 @@
+class Transfer::Generators::Sequel < Transfer::Generators::Base
+
+  def self.supports? klass
+    defined?(Sequel) && klass.ancestors.include?(Sequel::Model)
+  end
+
+  def before
+    klass.unrestrict_primary_key
+    @strict_param_setting = klass.strict_param_setting
+    klass.strict_param_setting = false
+    klass.set_restricted_columns if @restricted_columns = klass.restricted_columns
+    klass.set_allowed_columns if @allowed_columns = klass.allowed_columns
+  end
+
+  def after
+    klass.restrict_primary_key
+    klass.strict_param_setting = @strict_param_setting
+    klass.set_restricted_columns *@restricted_columns if @restricted_columns
+    klass.set_allowed_columns *@allowed_columns if @allowed_columns
+  end
+
+  def column_present? name
+    super name || klass.columns.include?(name.to_sym)
+  end
+
+  def transaction &block
+    klass.db.transaction :savepoint => true, &block
+  end
+
+  def create attributes, row, options={}, callbacks={}
+    model = klass.new attributes
+    model.instance_exec(row, &callbacks[:before_save]) if callbacks[:before_save]
+
+    save_options = options.select{|key| key == :validate }
+    save_options[:raise_on_failure] = true
+
+    model.save save_options
+    model.instance_exec(row, &callbacks[:after_save]) if callbacks[:after_save]
+    options[:success].call(model, row) if options[:success]
+    model
+  rescue Exception => e
+    options[:failure].call(model, row, e) if options[:failure]
+    raise Sequel::Rollback if options[:failure_strategy] == :rollback
+    model
+  end
+
+end

data/lib/transfer/transferer.rb

@@ -0,0 +1,90 @@
+class Transfer::Transferer
+  attr_reader :dataset, :klass
+
+  def initialize dataset, klass, &block
+    @dataset = dataset
+    @klass = klass
+    block.arity == 1 ? yield(self) : instance_eval(&block) if block_given?
+  end
+
+  def columns
+    @columns ||= dataset.columns.each_with_object({}){|i,hash| hash[i]=i if generator.column_present? i }
+  end
+
+  def build_attributes source
+    attrs = {}
+    columns.each do |name, value|
+      attrs[name] = case value
+        when Proc
+          value.call source
+        when Symbol
+          source[value]
+        else
+          value
+        end
+    end
+    attrs
+  end
+
+  def method_missing symbol, *args, &block
+    add_column symbol, block_given? ? block : args[0]
+  end
+
+  def process options = {}
+    generator.before
+    options[:before].call(klass, dataset) if options[:before]
+    generator.transaction do
+      dataset.each do |row|
+        attributes = build_attributes row
+        generator.create attributes, row, options, callbacks
+      end
+    end
+    options[:after].call(klass, dataset) if options[:after]
+    generator.after
+  end
+
+  def callbacks
+    @callbacks ||= {}
+  end
+
+  def before_save &block
+    callbacks[:before_save] = block
+  end
+
+  def after_save &block
+    callbacks[:after_save] = block
+  end
+
+  # def failure &block
+    # callbacks[:failure] = block
+  # end
+
+  def generator
+    @generator ||= GENERATORS.detect{|g| g.supports? klass}.new klass
+  end
+
+  private
+
+  GENERATORS = [
+    Transfer::Generators::Sequel,
+    Transfer::Generators::ActiveRecord,
+    Transfer::Generators::Mongoid,
+    Transfer::Generators::Base
+  ]
+
+
+  def add_column key, value
+    raise ArgumentError.new("method ##{key} in class #{klass} is not defined!") unless generator.column_present?(key)
+    raise ArgumentError.new("source column ##{value} not exists!") if value.instance_of?(Symbol) and !dataset.columns.include?(value)
+    columns[key] = value
+  end
+
+  def only *args
+    columns.delete_if {|key| !args.include?(key) }
+  end
+
+  def except *args
+    columns.delete_if {|key| args.include?(key) }
+  end
+
+end

data/lib/transfer/version.rb

@@ -0,0 +1,3 @@
+module Transfer
+  VERSION = "0.0.1"
+end

data/lib/transfer.rb

@@ -0,0 +1,50 @@
+require "transfer/version"
+
+
+module Transfer
+  extend self
+  autoload :Config, 'transfer/config'
+  autoload :Transferer, 'transfer/transferer'
+
+  module Generators
+    autoload :Sequel, 'transfer/generators/sequel'
+    autoload :ActiveRecord, 'transfer/generators/active_record'
+    autoload :Mongoid, 'transfer/generators/mongoid'
+    autoload :Base, 'transfer/generators/base'
+  end
+
+  def configure name = :default, &block
+    config = Config.new
+    yield config if block_given?
+    configs[name] = config
+  end
+
+  def configs
+    @configs ||= Hash.new {|hash, key| raise "config #{key} not exists" }
+  end
+end
+
+
+def transfer *args, &block
+  raise ArgumentError if args.length == 0
+
+  case args[0]
+  when Symbol, String
+    args[0] = Transfer.configs[args[0].to_sym]
+    transfer *args, &block
+  when Hash
+    transfer :default, *args, &block
+  when Transfer::Config
+    raise ArgumentError.new("second argument should be Hash!") unless args[1].instance_of?(Hash)
+    config, options = args[0], args[1]
+    process_keys = [:validate, :failure_strategy]
+    process_options = config.process_options.merge options.select{|key| process_keys.include?(key) }
+    sources = options.select{|key| !process_keys.include?(key) }
+    sources.each do |key, value|
+      dataset = config.connection[key]
+      transferer = Transfer::Transferer.new dataset, value, &block
+      transferer.process process_options
+    end
+  end
+
+end

data/spec/fabricators/user_fabricator.rb

@@ -0,0 +1,9 @@
+Fabricator :johnny_hollow, :from => :source_user do
+  fname "Johnny"
+  lname "Hollow"
+end
+
+Fabricator :olivia_lufkin, :from => :source_user do
+  fname "Olivia"
+  lname "Lufkin"
+end

data/spec/spec_helper.rb

@@ -0,0 +1,45 @@
+require 'rspec'
+require 'rr'
+require 'fabrication'
+require 'transfer'
+require 'database_cleaner'
+
+Dir["./spec/support/**/*.rb"].each {|f| require f}
+
+module TransfererHelper
+  def save_failure entity
+    if entity.kind_of?(Class)
+      instance_of(entity).save! { raise "force exception!" }
+      instance_of(entity).save  { raise "force exception!" }
+    else
+      stub(entity).save! { raise "force exception!" }
+      stub(entity).save  { raise "force exception!" }
+    end
+  end
+end
+
+RSpec.configure do |c|
+  c.mock_with :rr
+  c.include TransfererHelper
+
+  c.before :suite do
+    DatabaseCleaner[:mongoid].strategy = :truncation
+    DatabaseCleaner[:active_record].strategy = :transaction
+  end
+
+  c.around do |example|
+    Sequel.transaction Sequel::DATABASES, :rollback => :always do
+      example.run
+    end
+  end
+
+  c.before :each do
+    DatabaseCleaner.start
+  end
+
+  c.after :each do
+    DatabaseCleaner.clean
+  end
+end
+
+

data/spec/support/active_record.rb

@@ -0,0 +1,42 @@
+require 'active_record'
+
+ActiveRecord::Base.establish_connection :adapter => 'sqlite3', :database => ':memory:'
+ActiveRecord::Migration.verbose = false
+
+
+class ActiveRecordUserMigration < ActiveRecord::Migration
+  def self.up
+    create_table :active_record_users, :force => true do |t|
+      t.string :first_name
+      t.string :last_name
+      t.string :full_name
+      t.string :before_save_value
+      t.string :protected_value
+    end
+  end
+
+  def self.down
+    drop_table :active_record_users
+  end
+end
+
+ActiveRecordUserMigration.up
+
+
+class ActiveRecordUser < ActiveRecord::Base
+  set_table_name "active_record_users"
+  attr_accessor :dynamic_value
+  attr_protected :protected_value
+end
+
+class ActiveRecordUserWithFalseValidation < ActiveRecord::Base
+  set_table_name "active_record_users"
+  attr_accessor :dynamic_value
+  attr_protected :protected_value
+
+  validate :fake
+
+  def fake
+    errors.add :fake, 'fake'
+  end
+end

data/spec/support/mongoid.rb

@@ -0,0 +1,38 @@
+require 'mongoid'
+
+Mongoid.configure do |config|
+  config.master = Mongo::Connection.new.db("godfather")
+end
+
+class MongoidUser
+  include Mongoid::Document
+
+  field :first_name, :type => String
+  field :last_name, :type => String
+  field :full_name, :type => String
+  field :before_save_value, :type => String
+  field :protected_value, :type => String
+
+  attr_accessor :dynamic_value
+  attr_protected :protected_value
+end
+
+
+class MongoidUserWithFalseValidation
+  include Mongoid::Document
+
+  field :first_name, :type => String
+  field :last_name, :type => String
+  field :full_name, :type => String
+  field :before_save_value, :type => String
+  field :protected_value, :type => String
+
+  attr_accessor :dynamic_value
+  attr_protected :protected_value
+
+  validate :fake
+
+  def fake
+    errors.add :fake, 'fake'
+  end
+end

data/spec/support/sequel.rb

@@ -0,0 +1,45 @@
+require 'sequel'
+
+Sequel.extension :migration
+
+DESTINATION_DB = Sequel.sqlite
+
+class SequelUserMigration < Sequel::Migration
+  def up
+    create_table! :sequel_users do
+      primary_key :id
+      String :first_name
+      String :last_name
+      String :full_name
+      String :before_save_value
+      String :protected_value
+    end
+  end
+  def down
+    drop_table :sequel_users
+  end
+end
+
+
+SequelUserMigration.apply DESTINATION_DB, :up
+
+
+class SequelUser < Sequel::Model
+  attr_accessor :dynamic_value
+  set_restricted_columns :protected_value
+end
+
+
+class SequelUserWithFalseValidation < Sequel::Model(:sequel_users)
+  attr_accessor :dynamic_value
+  set_restricted_columns :protected_value
+
+  def validate
+    super
+    errors.add(:fake, 'fake error')
+  end
+end
+
+
+SequelUser.db = DESTINATION_DB
+SequelUserWithFalseValidation.db = DESTINATION_DB