trucker 0.1.0

data/BACKGROUND.markdown

@@ -0,0 +1,236 @@
+Background
+==========
+
+Trucker is based on a migration technique using LegacyModels first pioneered by Dave Thomas.
+
+Sharing External ActiveRecord Connections
+http://pragdave.blogs.pragprog.com/pragdave/2006/01/sharing_externa.html
+
+Using this, I've developed a set of helpers for migrating code.
+
+- /app/models/legacy/
+- /app/models/legacy/legacy_base.rb
+- /app/models/legacy/legacy_model.rb
+- /config/database.yml
+- /config/environment.rb
+- /lib/migration_helper.rb
+- /lib/tasks/migrate.rake
+
+
+
+/app/models/legacy/
+===================
+
+This folder will contain the base Legacy model, and all subclasses.
+
+
+/app/models/legacy/legacy_base.rb
+=================================
+
+This is the base Legacy model which connects to the legacy database and handles the migration.
+
+    class LegacyBase < ActiveRecord::Base
+      self.abstract_class = true
+      establish_connection "legacy"
+  
+      def migrate
+        new_record = self.class.to_s.gsub(/Legacy/,'::').constantize.new(map)
+        new_record[:id] = self.id
+        new_record.save
+      end
+
+    end
+
+
+/app/models/legacy/legacy_model.rb
+=================================
+
+This is a sample Legacy subclass, which specifies the legacy model name and defines a map of old field names to new field names. All Legacy models are stored in /app/models/legacy to keep your main app model namespace unaffected.
+
+    class LegacyModel < LegacyBase
+      set_table_name "model"
+
+      def map
+        {
+          :make => self.car_company.squish,
+          :model => self.car_name.squish
+        }
+      end
+
+    end
+
+
+
+/config/environment.rb
+======================
+
+We need to update the app environment so we can load the legacy models correctly.
+
+    Rails::Initializer.run do |config|
+      config.load_paths += %W( #{RAILS_ROOT}/app/models/legacy )
+    end
+
+
+
+/config/database.yml
+====================
+
+We need to add a custom database adapter so that we can connect to our legacy database.
+
+By convention, I've used APPNAME_legacy for my legacy databases, but you can easily customize this.
+
+    legacy:
+      adapter: mysql
+      database: APPNAME_legacy
+      username: root
+      password:
+
+
+
+/app/models/legacy/legacy_base.rb
+=================================
+
+This model connects to our legacy database, and provides a migration method.
+
+    class LegacyBase < ActiveRecord::Base
+      self.abstract_class = true
+      establish_connection "legacy"
+  
+      def migrate
+        new_record = self.class.to_s.gsub(/Legacy/,'::').constantize.new(map)
+        new_record[:id] = self.id
+        new_record.save
+      end
+
+    end
+
+
+
+/lib/migration_helper.rb
+========================
+
+This helper is used by the rake task to manage the actual migration process.
+
+    def migrate(name, options={})
+      # Grab custom entity label if present
+      label = options.delete(:label) if options[:label]
+  
+      unless options[:helper]
+    
+        # Grab model to migrate
+        model = name.to_s.singularize.capitalize
+    
+        # Wipe out existing records
+        model.constantize.delete_all
+
+        # Status message
+        status = "Migrating "
+        status += "#{number_of_records || "all"} #{label || name}"
+        status += " after #{offset_for_records}" if offset_for_records
+    
+        # Set import counter
+        counter = 0
+        counter += offset_for_records if offset_for_records
+        total_records = "Legacy#{model}".constantize.find(:all).size
+    
+        # Start import
+        "Legacy#{model}".constantize.find(:all, with(options)).each do |record|
+          counter += 1
+          puts status + " (#{counter}/#{total_records})"
+          record.migrate
+        end
+      else
+        eval options[:helper].to_s
+      end
+    end
+
+    def with(options={})
+      {:limit => number_of_records, :offset => offset_for_records}.merge(options)
+    end
+
+    def number_of_records
+      nil || ENV['limit'].to_i if ENV['limit'].to_i > 0
+    end
+
+    def offset_for_records
+      nil || ENV['offset'].to_i if ENV['offset'].to_i > 0
+    end
+
+  Available options include offset and limit:
+
+    rake db:migrate:architects limit=1000
+    rake db:migrate:architects limit=1000 offset=2000
+    
+
+
+  /lib/tasks/migrate.rake
+  ========================
+
+  This is the basic rake task for migrating legacy data. With a sample model, just add a new migration task with the pluralized name of your existing model. For more complicated migrations, the migrate method supports a helper method which will override the default migration behavior and allow you to do a highly customized migration.
+
+    require 'migration_helper'
+
+    namespace :db do
+      namespace :migrate do
+    
+        desc 'Migrates architects'
+        task :architects => :environment do
+          migrate :architects
+        end
+    
+        desc 'Migrates theaters'
+        task :architects => :environment do
+          migrate :theaters, :helper => :migrate_theaters
+        end
+
+      end
+    end
+
+    def migrate_theaters
+
+      # Delete all theaters if delete_all=true
+      Theater.delete_all if ENV['delete_all']
+
+      # Set default conditions
+      conditions = ["status_publish = 'Yes' AND location_address1 != '' AND location_address1 IS NOT NULL"]
+
+      # Set counters to monitor migration
+      success, failure, skipped = 0, 0, 0
+
+      # Count number of theaters
+      start = Theater.count
+
+      # Migrate theaters
+      puts "\nMigrating #{number_of_records || "all"} theaters #{"after #{offset_for_records}\n\n" if offset_for_records}"
+      LegacyTheater.find(:all, with(:conditions => conditions)).each_with_index do |record, i|
+
+        # Migrate theater
+        new_record = record.migrate
+        message = "#{new_record.name} (#{record.id})"
+
+        if Theater.exists?(record.id)
+          puts "#{i+1} SKIP #{message}\n\n"
+          skipped += 1
+        elsif new_record.save
+          puts "#{i+1} PASS #{message}\n\n"
+          success += 1
+        else
+          puts "#{i+1} FAIL #{message}\n#{new_record.inspect}\n\n"
+          failure += 1
+        end
+  
+        # Archive old theater data
+        archive_old_theater_data(record)
+
+      end
+
+      # Count number of theaters
+      finish = Theater.count
+
+      # Batch stats
+      percentage = (failure.to_f / (skipped + success + failure).to_f).to_f * 100
+      puts "BATCH: #{number_of_records || "all"} theaters => #{success} passed, #{failure} failed (#{percentage.truncate}%), #{skipped} skipped (already imported)"
+
+    end
+
+

data/INSTALL.markdown

@@ -0,0 +1,30 @@
+
+Here are some imaginary install instructions.
+
+1. Install the trucker gem and add to your gem config block.
+
+2. Generate the basic trucker files
+
+    script/generate truck
+
+    - Add legacy adapter to database.yml
+    - Add legacy base class
+    - (Optionally) Add legacy sub classes for all existing models
+    - Add app/models/legacy to load path in Rails Initializer config block
+    - Generate sample migration task (using pluralized model names)
+  
+3. Update database.yml with legacy database info
+
+4. Run rake db:create:all to create the legacy database
+
+5. Import your legacy database.
+   (Make sure the legacy adapter is using the correct name.)
+
+6. Update legacy model table names as needed
+
+7. Update legacy model field mappings as needed
+
+8. Start migrating!
+   rake db:migrate:models
+
+ 

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Patrick Crowley and Rob Kaufman
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,17 @@
+= Trucker
+
+Trucker is a gem for migrating legacy data into a Rails app.
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 Patrick Crowley and Rob Kaufman. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,47 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "trucker"
+    gem.summary = %Q{Bring your legacy along}
+    gem.description = %Q{Trucker is a gem for migrating legacy data into a Rails app}
+    gem.email = "patrick@mokolabs.com"
+    gem.homepage = "http://github.com/mokolabs/trucker"
+    gem.authors = ["Patrick Crowley and Rob Kaufman"]
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_development_dependency "yard", ">= 0"
+    gem.files = FileList["[A-Z]*", "{generators,lib,rails}/**/*"]
+    
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+begin
+  require 'yard'
+  YARD::Rake::YardocTask.new
+rescue LoadError
+  task :yardoc do
+    abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
+  end
+end

data/VERSION.yml

@@ -0,0 +1,5 @@
+--- 
+:major: 0
+:build: 
+:minor: 1
+:patch: 0

data/generators/truck/USAGE

@@ -0,0 +1 @@
+script/generate truck

data/generators/truck/lib/insert_commands.rb

@@ -0,0 +1,76 @@
+# Mostly pinched from http://github.com/ryanb/nifty-generators/tree/master
+
+Rails::Generator::Commands::Base.class_eval do
+  def file_contains?(relative_destination, line)
+    File.read(destination_path(relative_destination)).include?(line)
+  end
+end
+
+Rails::Generator::Commands::Create.class_eval do
+  def insert_before(file, line, stop='^(class|module) .+$')
+    logger.insert "#{line} into #{file}"
+    unless options[:pretend] || file_contains?(file, line)
+      gsub_file file, /^#{stop}/ do |match|
+        "#{line}\n#{match}"
+      end
+    end
+  end
+  
+  def insert_after(file, line, stop='^(class|module) .+$')
+    logger.insert "#{line} into #{file}"
+    @run = false
+    unless options[:pretend] || file_contains?(file, line)
+      gsub_file file, /#{stop}/ do |match|
+        @run = true
+        "#{match}\n  #{line}"
+      end
+      raise "Append Key Not Found, was looking for #{stop}" unless @run
+    end
+  end
+  
+  def append(file, line)
+    logger.insert "added legacy adapter to end of database.yml"
+    unless options[:pretend] || file_contains?(file, line)
+      File.open(file, "a") do |file|
+        file.write("\n" + line)
+      end
+    end
+  end
+  
+end
+
+Rails::Generator::Commands::Destroy.class_eval do
+  def insert_before(file, line, stop='')
+    logger.remove "#{line} from #{file}"
+    unless options[:pretend]
+      gsub_file file, "\n  #{line}", ''
+    end
+  end
+
+  def insert_after(file, line, stop='')
+    logger.remove "#{line} from #{file}"
+    unless options[:pretend]
+      gsub_file file, "\n  #{line}", ''
+    end
+  end
+  
+  def append(file, line)
+    logger.insert "added legacy adapter to end of database.yml"
+  end
+
+end
+
+Rails::Generator::Commands::List.class_eval do
+  def insert_before(file, line, stop='')
+    logger.insert "#{line} into #{file}"
+  end
+  
+  def insert_after(file, line, stop='')
+    logger.insert "#{line} into #{file}"
+  end
+  
+  def append(file, line)
+    logger.insert "added legacy adapter to end of database.yml"
+  end
+  
+end

data/generators/truck/templates/legacy_base.rb

@@ -0,0 +1,11 @@
+class LegacyBase < ActiveRecord::Base
+  self.abstract_class = true
+  establish_connection "legacy"
+  
+  def migrate
+    new_record = self.class.to_s.gsub(/Legacy/,'::').constantize.new(map)
+    new_record[:id] = self.id
+    new_record.save
+  end
+
+end

data/generators/truck/templates/legacy_model.erb

@@ -0,0 +1,11 @@
+class Legacy<%= model_name.classify %> < LegacyBase
+  set_table_name "<%= model_name.pluralize %>"
+
+  def map
+    {
+      :field_one => self.old_field_one.squish,
+      :field_two => self.old_field_two.squish
+    }
+  end
+
+end

data/generators/truck/templates/legacy_task.erb

@@ -0,0 +1,23 @@
+require 'trucker'
+include Trucker
+
+namespace :db do
+  namespace :migrate do
+
+    <%- unless legacy_models.size > 0 -%>
+    # desc 'Migrates products'
+    # task :products => :environment do
+    #   migrate :products
+    # end
+    <%- end -%>
+    
+    <%- legacy_models.each do |model_name| -%>
+    desc 'Migrates <%= model_name.pluralize.downcase %>'
+    task :<%= model_name.pluralize.downcase %> => :environment do
+      Trucker.migrate :<%= model_name.pluralize.downcase %>
+    end
+    
+    <%- end -%>
+
+  end
+end

data/generators/truck/truck_generator.rb

@@ -0,0 +1,36 @@
+require File.expand_path(File.dirname(__FILE__) + "/lib/insert_commands.rb")
+
+class TruckGenerator < Rails::Generator::Base
+
+  def manifest
+    record do |m|
+      @legacy_models = Dir.glob(RAILS_ROOT + '/app/models/*.rb').collect { |model_path| File.basename(model_path).gsub('.rb', '') }
+
+      m.directory 'app/models/legacy'
+      m.file 'legacy_base.rb', 'app/models/legacy/legacy_base.rb'
+
+      @legacy_models.each do |model_name|
+        m.template 'legacy_model.erb', "app/models/legacy/legacy_#{model_name.downcase}.rb", :assigns => { :model_name => model_name }
+      end
+
+      m.directory 'lib/tasks'
+      m.template 'legacy_task.erb', 'lib/tasks/legacy.rake', :assigns => { :legacy_models => @legacy_models }
+      
+      snippet = <<EOS
+legacy:
+  adapter: mysql
+  database: #{RAILS_ROOT.split('/').last}_legacy
+  encoding: utf8
+  username:
+  password:
+EOS
+
+      m.append "config/database.yml", snippet
+      
+      snippet = 'config.load_paths += %W( #{RAILS_ROOT}/app/models/legacy )'
+      m.insert_after "config/environment.rb", snippet, '^Rails::Initializer\.run do.+$'
+      
+    end
+  end
+
+end

data/lib/trucker.rb

@@ -0,0 +1,50 @@
+module Trucker
+
+  def self.migrate(name, options={})
+    # Grab custom entity label if present
+    label = options.delete(:label) if options[:label]
+
+    unless options[:helper]
+  
+      # Grab model to migrate
+      model = name.to_s.singularize.capitalize
+  
+      # Wipe out existing records
+      model.constantize.delete_all
+
+      # Status message
+      status = "Migrating "
+      status += "#{number_of_records || "all"} #{label || name}"
+      status += " after #{offset_for_records}" if offset_for_records
+  
+      # Set import counter
+      counter = 0
+      counter += offset_for_records if offset_for_records
+      total_records = "Legacy#{model}".constantize.find(:all).size
+  
+      # Start import
+      "Legacy#{model}".constantize.find(:all, with(options)).each do |record|
+        counter += 1
+        puts status + " (#{counter}/#{total_records})"
+        record.migrate
+      end
+    else
+      eval options[:helper].to_s
+    end
+  end
+
+  protected
+
+    def self.with(options={})
+      {:limit => number_of_records, :offset => offset_for_records}.merge(options)
+    end
+
+    def self.number_of_records
+      nil || ENV['limit'].to_i if ENV['limit'].to_i > 0
+    end
+
+    def self.offset_for_records
+      nil || ENV['offset'].to_i if ENV['offset'].to_i > 0
+    end
+  
+end

data/rails/init.rb

@@ -0,0 +1 @@
+require 'trucker'

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'trucker'
+require 'spec'
+require 'spec/autorun'
+
+Spec::Runner.configure do |config|
+  
+end

data/spec/trucker_spec.rb

@@ -0,0 +1,7 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "Trucker" do
+  it "fails" do
+    fail "hey buddy, you should probably rename this file and start specing for real"
+  end
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification 
+name: trucker
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Patrick Crowley and Rob Kaufman
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-07-15 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 13
+        segments: 
+        - 1
+        - 2
+        - 9
+        version: 1.2.9
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: yard
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+description: Trucker is a gem for migrating legacy data into a Rails app
+email: patrick@mokolabs.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- BACKGROUND.markdown
+- INSTALL.markdown
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION.yml
+- generators/truck/USAGE
+- generators/truck/lib/insert_commands.rb
+- generators/truck/templates/legacy_base.rb
+- generators/truck/templates/legacy_model.erb
+- generators/truck/templates/legacy_task.erb
+- generators/truck/truck_generator.rb
+- lib/trucker.rb
+- rails/init.rb
+- spec/spec_helper.rb
+- spec/trucker_spec.rb
+has_rdoc: true
+homepage: http://github.com/mokolabs/trucker
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Bring your legacy along
+test_files: 
+- spec/spec_helper.rb
+- spec/trucker_spec.rb