forklift_etl 1.0.8

data/spec/spec_helper.rb

@@ -0,0 +1,30 @@
+#############
+## WARNING ##
+#############
+
+# THIS TEST SUITE IS VERY MEAN TO MYSQL AND ELASTICSEARCH
+# IT *WILL* DELETE ANY CONTENT IN THE TEST DBs
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+APP_DIR ||= File.expand_path('../../', __FILE__)
+
+require 'forklift/forklift'
+require 'awesome_print'
+require 'rspec'
+require 'fileutils'
+
+ENV["FORKLIFT_RUN_ALL_STEPS"] = 'true'
+
+Dir["#{APP_DIR}/spec/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+
+  config.before(:all) do
+    piddir = "#{File.dirname(__FILE__)}/pid"
+    FileUtils.rmdir(piddir) if File.exists?(piddir)
+    SpecSeeds.setup_mysql
+    SpecSeeds.setup_elasticsearch
+  end
+
+end

data/spec/support/dumps/elasticsearch/forklift_test.json

@@ -0,0 +1,7 @@
+[
+  {"id": 1, "user_id": 1, "product_id": 1, "viewed_at": 1396552251},
+  {"id": 2, "user_id": 1, "product_id": 2, "viewed_at": 1396552252},
+  {"id": 3, "user_id": 2, "product_id": 5, "viewed_at": 1396552253},
+  {"id": 4, "user_id": 2, "product_id": 5, "viewed_at": 1396552254},
+  {"id": 5, "user_id": 2, "product_id": 5, "viewed_at": 1396552255}
+]

data/spec/support/dumps/mysql/forklift_test_source_a.sql

@@ -0,0 +1,79 @@
+# Dump of table products
+# ------------------------------------------------------------
+
+DROP TABLE IF EXISTS `products`;
+
+CREATE TABLE `products` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) NOT NULL DEFAULT '',
+  `inventory` int(11) DEFAULT NULL,
+  `created_at` datetime NOT NULL,
+  `updated_at` datetime NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+LOCK TABLES `products` WRITE;
+
+INSERT INTO `products` (`id`, `name`, `inventory`, `created_at`, `updated_at`)
+VALUES
+	(1,'car',10,'2014-04-03 11:45:51','2014-04-03 11:45:51'),
+	(2,'boat',3,'2014-04-03 11:45:52','2014-04-03 11:45:52'),
+	(3,'bus',5,'2014-04-03 11:45:54','2014-04-03 11:45:54'),
+	(4,'motorcycle',23,'2014-04-03 11:45:56','2014-04-03 11:45:56'),
+	(5,'hang_glider',2,'2014-04-03 11:46:19','2014-04-03 11:46:19');
+
+UNLOCK TABLES;
+
+
+# Dump of table sales
+# ------------------------------------------------------------
+
+DROP TABLE IF EXISTS `sales`;
+
+CREATE TABLE `sales` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `user_id` int(11) NOT NULL,
+  `product_id` int(11) NOT NULL,
+  `timestamp` datetime NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+LOCK TABLES `sales` WRITE;
+
+INSERT INTO `sales` (`id`, `user_id`, `product_id`, `timestamp`)
+VALUES
+	(1,1,1,'2014-04-03 11:47:11'),
+	(2,1,2,'2014-04-03 11:47:11'),
+	(3,4,5,'2014-04-03 11:47:12'),
+	(4,4,4,'2014-04-03 11:47:25'),
+	(5,5,5,'2014-04-03 11:47:26');
+
+UNLOCK TABLES;
+
+
+# Dump of table users
+# ------------------------------------------------------------
+
+DROP TABLE IF EXISTS `users`;
+
+CREATE TABLE `users` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `email` varchar(255) NOT NULL DEFAULT '',
+  `first_name` varchar(255) NOT NULL DEFAULT '',
+  `last_name` varchar(255) NOT NULL DEFAULT '',
+  `created_at` datetime NOT NULL,
+  `updated_at` datetime NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+LOCK TABLES `users` WRITE;
+
+INSERT INTO `users` (`id`, `email`, `first_name`, `last_name`, `created_at`, `updated_at`)
+VALUES
+	(1,'evan@example.com','Evan','T','2014-04-03 11:40:12','2014-04-03 11:39:28'),
+	(2,'pablo@example.com','Pablo ','J','2014-04-03 11:41:08','2014-04-03 11:41:08'),
+	(3,'kevin@example.com','Kevin','B','2014-04-03 11:41:10','2014-04-03 11:41:10'),
+	(4,'brian@example.com','Brian','L','2014-04-03 11:41:12','2014-04-03 11:41:12'),
+	(5,'aaront@example.com','Aaron','B','2014-04-03 11:41:13','2014-04-03 11:41:13');
+
+UNLOCK TABLES;

data/spec/support/dumps/mysql/forklift_test_source_b.sql

@@ -0,0 +1,23 @@
+# Dump of table admin_notes
+# ------------------------------------------------------------
+
+DROP TABLE IF EXISTS `admin_notes`;
+
+CREATE TABLE `admin_notes` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `user_id` int(11) NOT NULL,
+  `note` text NOT NULL,
+  `created_at` datetime NOT NULL,
+  `updated_at` datetime NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+LOCK TABLES `admin_notes` WRITE;
+
+INSERT INTO `admin_notes` (`id`, `user_id`, `note`, `created_at`, `updated_at`)
+VALUES
+	(1,1,'User 1 called customer support\n','2014-04-03 11:50:25','2014-04-03 11:50:25'),
+	(2,2,'User 2 called customer support','2014-04-03 11:50:26','2014-04-03 11:50:26'),
+	(3,5,'User 5 returned the purchase','2014-04-03 11:50:28','2014-04-03 11:50:28');
+
+UNLOCK TABLES;

data/spec/support/spec_client.rb

@@ -0,0 +1,30 @@
+require 'yaml'
+require 'erb'
+
+class SpecClient
+
+  def self.load_config(file)
+    YAML.load(ERB.new(File.read(file)).result)
+  end
+
+  def self.mysql(name)
+    file = File.join(File.dirname(__FILE__), '..', 'config', 'connections', 'mysql', "#{name}.yml")
+    config = self.load_config(file)
+    db = config[:database]
+    config.delete(:database)
+    connection = ::Mysql2::Client.new(config)
+    begin 
+      connection.query("use `#{db}`")
+    rescue Exception => e
+      puts "#{e} => will create new databse #{db}"
+    end
+    connection
+  end
+
+  def self.elasticsearch(name)
+    file = File.join(File.dirname(__FILE__), '..', 'config', 'connections', 'elasticsearch', "#{name}.yml")
+    config = self.load_config(file)
+    ::Elasticsearch::Client.new(config)
+  end
+
+end

data/spec/support/spec_plan.rb

@@ -0,0 +1,15 @@
+class SpecPlan
+  def self.config
+    return {
+      :project_root => File.join(Dir.pwd, 'spec'),
+      :logger => {
+        :stdout => false,
+        :debug => false,
+      },
+    }
+  end
+
+  def self.new
+    return Forklift::Plan.new(self.config)
+  end
+end

data/spec/support/spec_seeds.rb

@@ -0,0 +1,69 @@
+require 'json'
+
+class SpecSeeds
+
+  def self.setup_mysql
+    mysql_connections         = []
+    mysql_databases           = []
+
+    files = Dir["#{File.dirname(__FILE__)}/../config/connections/mysql/*.yml"]
+    files.each do |f|
+      name = f.split('/').last.gsub('.yml','')
+      mysql_connections << ::SpecClient.mysql(name)
+      mysql_databases << name
+    end
+
+    i = 0
+    while i < mysql_connections.count
+      conn = mysql_connections[i]
+      db   = mysql_databases[i]
+      seed = File.join(File.dirname(__FILE__), '..', 'support', 'dumps', 'mysql', "#{db}.sql")
+      conn.query("drop database if exists `#{db}`")
+      conn.query("create database `#{db}`")
+      conn.query("use `#{db}`")
+      if File.exists? seed
+        lines = File.read(seed).split(";")
+        lines.each do |line|
+          conn.query(line) if line[0] != "#"
+        end
+      end
+
+      i = i + 1
+    end
+  end
+
+  def self.setup_elasticsearch
+    elasticsearch_connections = []
+    elasticsearch_databases   = []
+
+    files = Dir["#{File.dirname(__FILE__)}/../config/connections/elasticsearch/*.yml"]
+    files.each do |f|
+      name = f.split('/').last.gsub('.yml','')
+      elasticsearch_connections << ::SpecClient.elasticsearch(name)
+      elasticsearch_databases << name
+    end
+
+    i = 0
+    while i < elasticsearch_connections.count
+      conn  = elasticsearch_connections[i]
+      index = elasticsearch_databases[i]
+      seed  = File.join(File.dirname(__FILE__), '..', 'support', 'dumps', 'elasticsearch', "#{index}.json")
+      conn.indices.delete({ :index => index }) if conn.indices.exists({ :index => index })
+      if File.exists? seed
+        lines = JSON.parse(File.read(seed))
+        lines.each do |line|
+          object = {
+            :index => index,
+            :body  => line,
+            :type  => 'forklift',
+            :id    => line['id']
+          }
+          conn.index object # assumes ES is setup to allow index creation on write
+        end
+        conn.indices.refresh({ :index => index })
+      end
+      i = i + 1
+    end
+  end
+
+end

data/spec/template/spec_email_template.erb

@@ -0,0 +1,4 @@
+## Your forklift email ##
+
+Total Users: <%= @total_users_count %>
+New Users: <%= @new_users_count %>

data/spec/unit/connection/mysql_spec.rb

@@ -0,0 +1,102 @@
+require 'spec_helper'
+require 'zlib'
+
+describe Forklift::Connection::Mysql do
+
+  describe "read/write utils" do
+    before(:each) do
+      SpecSeeds.setup_mysql
+    end
+
+    it "can read a list of tables" do
+      plan = SpecPlan.new
+      plan.do! {
+        source = plan.connections[:mysql][:forklift_test_source_a]
+        expect(source.tables).to include 'users'
+        expect(source.tables).to include 'products'
+        expect(source.tables).to include 'sales'
+      }
+    end
+
+    it "can delte a table" do 
+      plan = SpecPlan.new
+      table = "users"
+      plan.do! {
+        source = plan.connections[:mysql][:forklift_test_source_a]
+        expect(source.tables).to include 'users'
+        source.drop! table
+        expect(source.tables).to_not include 'users'
+      }
+    end
+
+    it "can count the rows in a table" do
+      plan = SpecPlan.new
+      table = "users"
+      plan.do! {
+        source = plan.connections[:mysql][:forklift_test_source_a]
+        expect(source.count(table)).to eql 5
+      }
+    end
+
+    it "can truncate a table (both with and without !)" do 
+      plan = SpecPlan.new
+      table = "users"
+      plan.do! {
+        source = plan.connections[:mysql][:forklift_test_source_a]
+        expect(source.count(table)).to eql 5
+        source.truncate! table
+        expect(source.count(table)).to eql 0
+        expect { source.truncate(table) }.to_not raise_error
+      }
+    end
+
+    it 'trunacte! will raise if the table does not exist' do
+      plan = SpecPlan.new
+      table = "other_table"
+      plan.do! {
+        source = plan.connections[:mysql][:forklift_test_source_a]
+        expect { source.truncate!(table) }.to raise_error(/Table 'forklift_test_source_a.other_table' doesn't exist/)
+      }
+    end
+
+    it "can get the columns of a table" do 
+      plan = SpecPlan.new
+      table = "sales"
+      plan.do! {
+        source = plan.connections[:mysql][:forklift_test_source_a]
+        expect(source.columns(table)).to include 'id' 
+        expect(source.columns(table)).to include 'user_id' 
+        expect(source.columns(table)).to include 'product_id' 
+        expect(source.columns(table)).to include 'timestamp' 
+      }
+    end
+
+    it "can create a mysqldump" do
+      dump = "/tmp/destination.sql.gz"
+      plan = SpecPlan.new
+      plan.do! {
+        source = plan.connections[:mysql][:forklift_test_source_a]
+        source.dump(dump)
+      }
+
+      expect(File.exists?(dump)).to eql true
+      contents = Zlib::GzipReader.new(StringIO.new(File.read(dump))).read  
+      expect(contents).to include "(1,'evan@example.com','Evan','T','2014-04-03 11:40:12','2014-04-03 11:39:28')"
+    end
+
+  end
+
+  describe "#safe_values" do
+    subject { described_class.new({}, {}) }
+
+    it "escapes one trailing backslash" do
+      values = ["foo\\"]
+      subject.send(:safe_values, values).should == "\"foo\\\\\""
+    end
+
+    it "escapes two trailing backslashes" do
+      values = ["foo\\\\"]
+      subject.send(:safe_values, values).should == "\"foo\\\\\\\\\""
+    end
+  end
+end

data/spec/unit/misc/email_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+require "email_spec"
+
+describe 'misc forklift core' do  
+  describe 'email' do
+    include EmailSpec::Helpers
+    include EmailSpec::Matchers
+
+    it "can send mail with an email template" do
+      plan = SpecPlan.new
+      plan.do! {
+        email_args = {
+          :to      => "YOU@FAKE.com",
+          :from    => "Forklift",
+          :subject => "Forklift has moved your database",
+        }
+        email_variables = {
+          :total_users_count => 10,
+          :new_users_count => 5,
+        }
+        email_template = "#{File.dirname(__FILE__)}/../../template/spec_email_template.erb"
+        @email = plan.mailer.send_template(email_args, email_template, email_variables).first
+      }
+      
+      @email.should deliver_to("YOU@FAKE.com")
+      @email.should have_subject(/Forklift has moved your database/)
+      @email.should have_body_text(/Your forklift email/) # base 
+      @email.should have_body_text(/Total Users: 10/) # template
+      @email.should have_body_text(/New Users: 5/) # template
+    end
+
+    it "can send mail with an attachment" do 
+      pending("how to test email attachments?")
+    end
+  end
+
+end

data/spec/unit/misc/pid_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe 'misc forklift core' do  
+
+  describe 'pidfile' do
+    it "can create a pidfile and will remove it when the plan is over" do
+      plan = SpecPlan.new
+      pid = "#{File.dirname(__FILE__)}/../../pid/pidfile"
+      expect(File.exists?(pid)).to eql false
+      plan.do! {
+        expect(File.exists?(pid)).to eql true
+        expect(File.read(pid).to_i).to eql Process.pid
+      }
+      expect(File.exists?(pid)).to eql false
+    end
+
+    it "will not run with an existing pidfile" do 
+      plan = SpecPlan.new
+      plan.pid.store!
+      expect { plan.do! }.to raise_error SystemExit
+      plan.pid.delete!
+    end 
+  end
+
+end

data/spec/unit/misc/step_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+describe 'misc forklift core' do  
+  describe 'steps' do
+
+    before(:each) do 
+      ENV['FORKLIFT_RUN_ALL_STEPS'] = 'false'
+    end
+
+    after(:each) do
+      ENV['FORKLIFT_RUN_ALL_STEPS'] = 'true'
+    end
+
+    it "will run all steps with no extra ARGV" do
+      plan = SpecPlan.new
+      plan.stub(:argv){ ['/path/to/plan'] }
+      steps_run = []
+      plan.do! {
+        plan.step("a"){ steps_run << 'a' }
+        plan.step("b"){ steps_run << 'b' }
+        plan.step("c"){ steps_run << 'c' }
+      }
+      expect(steps_run).to include 'a'
+      expect(steps_run).to include 'b'
+      expect(steps_run).to include 'c'
+    end
+
+    it "will only run steps named within ARGV" do
+      plan = SpecPlan.new
+      plan.stub(:argv){ ['/path/to/plan', 'a','c'] }
+      steps_run = []
+      plan.do! {
+        plan.step("a"){ steps_run << 'a' }
+        plan.step("b"){ steps_run << 'b' }
+        plan.step("c"){ steps_run << 'c' }
+      }
+      expect(steps_run).to include 'a'
+      expect(steps_run).to_not include 'b'
+      expect(steps_run).to include 'c'
+    end
+
+    it "won't run on a badly defined step" do
+      plan = SpecPlan.new
+      plan.stub(:argv){ ['/path/to/plan', 'missing_step'] }      
+      expect{
+        plan.do! {
+          plan.step("a"){ raise 'never should get here' }
+        }
+      }.to raise_error SystemExit
+    end
+  end
+
+end