forklift_etl 1.0.8

data/example/template/email.erb

@@ -0,0 +1,6 @@
+<h1>Your forklift email</h1>
+
+<ul>
+  <li><strong>Total Users</strong>: <%= @total_users_count %></li>
+  <li><strong>New Users</strong>: <%= @new_users_count %></li>
+</ul>

data/example/transformations/cleanup.sql

@@ -0,0 +1 @@
+ALTER TABLE `users` DROP `combined_name`;

data/example/transformations/combined_name.sql

@@ -0,0 +1,7 @@
+ALTER TABLE `users` ADD `combined_name` VARCHAR(255)  NULL DEFAULT NULL  AFTER `last_name`;
+
+UPDATE `users` SET `combined_name` = (
+  select CONCAT(first_name, " ", last_name)
+);
+
+CREATE INDEX combined_name ON users (combined_name);

data/example/transformations/email_suffix.rb

@@ -0,0 +1,20 @@
+class EmailSuffix
+
+  def do!(connection, forklift)
+    forklift.logger.log "collecting email suffixes..."
+
+    suffixes = {}
+    connection.read("select email from users"){|data|
+      data.each do |row|
+        part = row[:email].split('@').last
+        suffixes[part] = 0 if suffixes[part].nil?
+        suffixes[part] = suffixes[part] + 1
+      end
+    }
+
+    suffixes.each do |suffix, count|
+      forklift.logger.log " > #{suffix}: #{count}" if count > 5
+    end
+  end
+
+end

data/forklift_etl.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'forklift/version'
+
+Gem::Specification.new do |s|
+  s.name        = "forklift_etl"
+  s.version     = Forklift::VERSION
+  s.authors     = ["Evan Tahler"]
+  s.email       = ["evan@taskrabbit.com"]
+  s.homepage    = "https://github.com/taskrabbit/forklift"
+  s.summary     = %q{Forklift: Moving big databases around. A ruby ETL tool.}
+  s.description = %q{A collection of ETL tools and patterns for mysql and elasticsearch.}
+  s.license     = "MIT"
+
+  s.rubyforge_project = "forklift_etl"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "activesupport", '~> 4.0', ">= 4.0.0"
+  s.add_dependency "mysql2",        '~> 0.0', ">= 0.0.1"
+  s.add_dependency "elasticsearch", '~> 1.0', ">= 1.0.0"
+  s.add_dependency "pony",          '~> 1.0', ">= 1.0.0"
+  s.add_dependency "lumberjack",    '~> 1.0', ">= 1.0.0"
+end

data/lib/forklift/base/connection.rb

@@ -0,0 +1,72 @@
+module Forklift
+  module Base
+    class Connection
+
+      def initialize(config)
+        @config = config
+      end
+
+      def config
+        @config
+      end
+
+      def client
+        @client
+      end
+
+      def connect
+        # Will define @client
+        raise 'not implemented'
+      end
+
+      def disconnect
+        raise 'not implemented'
+      end
+
+      def read(query)
+        # will return an array of data rows
+        raise 'not implemented'
+      end
+
+      def write(data, collection)
+        # will write array data to collection (table)
+        raise 'not implemented'
+      end
+
+      def pipe
+        # when copying within the same connection, this method can be defined to speed things up
+        raise 'not implemented'
+      end
+
+      def exec(path)
+        begin
+          exec!(path)
+        rescue Exception => e
+          forklift.logger.log(e)
+        end
+      end
+
+      def exec!(path)
+        forklift.logger.log "Running script: #{path}"
+        extension = path.split(".").last
+        if(extension == "rb" || extension == "ruby")
+          exec_ruby(path)
+        else
+          exec_script(path)
+        end
+      end
+
+      def exec_ruby(path)
+        klass = forklift.utils.class_name_from_file(path)
+        require path
+        model = eval("#{klass}.new")
+        model.do!(self, forklift)
+      end
+
+      def exec_script(path)
+        raise 'not implemented'
+      end
+
+    end
+  end
+end

data/lib/forklift/base/logger.rb

@@ -0,0 +1,49 @@
+require 'lumberjack'
+
+module Forklift
+  module Base
+    class Logger
+
+      def initialize(forklift)
+        @forklift = forklift
+      end
+
+      def forklift
+        @forklift
+      end
+
+      def messages
+        @messages ||= []
+      end
+
+      def logger
+        log_dir = "#{forklift.config[:project_root]}/log"
+        @logger ||= ::Lumberjack::Logger.new("#{log_dir}/forklift.log", :buffer_size => 0)
+      end
+
+      def log(message, severity="info")
+        timed_message = "[Forklift @ #{Time.now}] #{message}"
+        puts timed_message unless forklift.config[:logger][:stdout] != true
+        logger.send(severity.to_sym, message) unless logger.nil?
+        messages << timed_message
+      end
+
+      def debug(message)
+        if forklift.config[:logger][:debug] == true
+          log("[debug] #{message}")
+        end
+      end
+
+      def emphatically(message)
+        log "" if message.length > 0
+        log "*** #{message} ***"
+        log ""
+      end
+
+      def fatal(message)
+        log "!!! #{message} !!!"
+      end
+
+    end
+    end
+end

data/lib/forklift/base/mailer.rb

@@ -0,0 +1,83 @@
+require 'pony'
+require 'erb'
+require 'active_support/core_ext/hash/keys'
+
+module Forklift
+  module Base
+    class Mailer
+
+      def initialize(forklift)
+        @forklift = forklift
+      end
+
+      # Public: Pull out the settings from config/email.yml.
+      #
+      # Returns a Hash with all symbolized keys.
+      def config
+        config_file = "#{forklift.config[:project_root]}/config/email.yml"
+        @config ||= forklift.utils.load_yml(config_file).deep_symbolize_keys
+      end
+
+      def forklift
+        @forklift
+      end
+
+      def message_defaults
+        {
+          :from => "Forklift",
+          :subject => "Forklift has moved your database @ #{Time.new}",
+          :body => "Forklift has moved your database @ #{Time.new}",
+        }
+      end
+
+      def send_template(args, template_file, variables, attachment_lines=[])
+        renderer = ERB.new(File.read(template_file))
+        binder = ERBBinding.new(variables)
+        body = renderer.result(binder.get_binding)
+        args[:body] = body
+        send(args, attachment_lines)
+      end
+
+      def send(args, attachment_lines=[])
+        params = message_defaults
+        [:to, :from, :subject, :body].each do |i|
+          params[i] = args[i] unless args[i].nil?
+        end
+        if attachment_lines.length > 0
+          params[:attachments] = {"log.txt" => attachment_lines.join("\r\n")}
+        end
+        deliver(params)
+      end
+
+      private
+
+      # Private: Actually deliver the message using Pony.
+      #
+      # Returns the raw email from Pony.
+      def deliver(params)
+        forklift.logger.log("Sending email via #{config[:via]}")
+        if params[:html_body].nil?
+          params[:html_body] = params[:body]
+          params.delete(:body)
+        end
+        params[:via] = config[:via].to_sym
+        params[:via_options] = config[:via_options]
+        Pony.mail(params)
+      end
+
+      class ERBBinding
+        def initialize(hash)
+          hash.each do |k,v|
+            v = v.gsub("'", " ") if v.class == String
+            instance_variable_set("@#{k}", v)
+          end
+        end
+
+        def get_binding
+          return binding()
+        end
+      end
+
+    end
+  end
+end

data/lib/forklift/base/pid.rb

@@ -0,0 +1,55 @@
+module Forklift
+  module Base
+    class Pid
+
+      def initialize(forklift)
+        @forklift = forklift
+      end
+
+      def forklift
+        @forklift
+      end
+
+      def pid_dir
+        "#{forklift.config[:project_root]}/pid"
+      end
+
+      def ensure_pid_dir
+        `mkdir -p #{pid_dir}`
+      end
+
+      def pidfile
+        "#{pid_dir}/pidfile"
+      end
+
+      def store!
+        forklift.logger.debug "Creating pidfile @ #{pidfile}"
+        ensure_pid_dir
+        File.open(pidfile, 'w') {|f| f << Process.pid}
+      end
+
+      def recall
+        ensure_pid_dir
+        IO.read(pidfile).to_i rescue nil
+      end
+
+      def delete!
+        forklift.logger.debug "Removing pidfile @ #{pidfile}"
+        FileUtils.rm(pidfile) rescue nil
+      end
+
+      def safe_to_run?
+        return if recall.nil?
+        count = `ps -p #{recall} | wc -l`.to_i
+        if count >= 2
+          forklift.logger.fatal "This application is already running (pidfile) #{recall}. Exiting now"
+          exit(1)
+        else
+          forklift.logger.log "Clearing old pidfile from previous process #{recall}"
+          delete!
+        end
+      end
+
+    end
+  end
+end

data/lib/forklift/base/utils.rb

@@ -0,0 +1,23 @@
+require 'yaml'
+require 'erb'
+
+module Forklift
+  module Base
+    class Utils
+
+      def load_yml(file)
+        YAML.load(ERB.new(File.read(file)).result)
+      end
+
+      def class_name_from_file(file)
+        klass = ""
+        words = file.split("/").last.split(".").first.split("_")
+        words.each do |word|
+          klass << word.capitalize
+        end
+        klass
+      end
+
+    end
+  end
+end

data/lib/forklift/forklift.rb

@@ -0,0 +1,19 @@
+require 'rubygems'
+
+module Forklift
+
+  lib = File.expand_path(File.dirname(__FILE__))
+
+  require "#{lib}/base/utils.rb"
+  require "#{lib}/base/pid.rb"
+  require "#{lib}/base/logger.rb"
+  require "#{lib}/base/mailer.rb"
+  require "#{lib}/base/connection.rb"
+
+  Dir["#{lib}/transports/*.rb"].each {|file| require file }
+  Dir["#{lib}/patterns/*.rb"].each {|file| require file }
+  Dir["#{Dir.pwd}/transports/*.rb"].each {|file| require file } if File.directory?("#{Dir.pwd}/transports")
+  Dir["#{Dir.pwd}/patterns/*.rb"].each {|file| require file } if File.directory?("#{Dir.pwd}/patterns")
+
+  require "#{lib}/plan.rb"
+end

data/lib/forklift/patterns/elasticsearch_patterns.rb

@@ -0,0 +1,7 @@
+module Forklift
+  module Patterns
+    class Elasticsearch
+
+    end
+  end
+end

data/lib/forklift/patterns/mysql_patterns.rb

@@ -0,0 +1,87 @@
+module Forklift
+  module Patterns
+    class Mysql
+
+      def self.pipe(source, from_table, destination, to_table)
+        start = Time.new.to_i
+        from_db = source.current_database 
+        to_db = destination.current_database 
+        source.forklift.logger.log("mysql pipe: `#{from_db}`.`#{from_table}` => `#{to_db}`.`#{to_table}`")
+        source.q("drop table if exists `#{to_db}`.`#{to_table}`")
+        source.q("create table `#{to_db}`.`#{to_table}` like `#{from_db}`.`#{from_table}`")
+        source.q("insert into `#{to_db}`.`#{to_table}` select * from `#{from_db}`.`#{from_table}`")
+        delta = Time.new.to_i - start
+        source.forklift.logger.log("  ^ moved #{destination.count(to_table, to_db)} rows in #{delta}s")
+      end
+
+      def self.incremental_pipe(source, from_table, destination, to_table, matcher=source.default_matcher, primary_key='id')
+        start = Time.new.to_i
+        from_db = source.current_database 
+        to_db = destination.current_database 
+        source.forklift.logger.log("mysql incremental_pipe: `#{from_db}`.`#{from_table}` => `#{to_db}`.`#{to_table}`")
+        source.q("create table if not exists `#{to_db}`.`#{to_table}` like `#{from_db}`.`#{from_table}`")
+
+        # Count the number of rows in to_table
+        original_count = source.count(to_table, to_db)
+
+        # Find the latest/max/newest timestamp from the final table
+        # in order to determine the last copied row.
+        latest_timestamp = source.max_timestamp(to_table, matcher, to_db)
+
+        # If to_table has existing rows, ensure none of them are "stale."
+        # A stale row in to_table means a previously copied row was
+        # updated in from_table, so let's delete it from the to_table
+        # so we can get a fresh copy of that row.
+        if original_count > 0
+          # Get the ids of rows in from_table that are newer than the newest row in to_table.
+          # Some of these rows could either be a) stale or b) new.
+          source.read("select `#{primary_key}` from `#{from_db}`.`#{from_table}` where `#{matcher}` > \"#{latest_timestamp}\" order by `#{matcher}`") do |stale_rows|
+            if stale_rows.length > 0
+              # Delete these ids from to_table.
+              # If the ids are stale, then they'll be deleted. If they're new, they won't exist, and nothing will happen.
+              stale_ids = stale_rows.map { |row| row[primary_key.to_sym] }.join(',')
+              source.q("delete from `#{to_db}`.`#{to_table}` where `#{primary_key}` in (#{stale_ids})")
+              source.forklift.logger.log("  ^ deleted up to #{stale_rows.length} stale rows from `#{to_db}`.`#{to_table}`")
+            end
+          end
+        end
+
+        # Do the insert into to_table
+        destination.q("insert into `#{to_db}`.`#{to_table}` select * from `#{from_db}`.`#{from_table}` where `#{matcher}` > \"#{latest_timestamp}\" order by `#{matcher}`")
+        delta = Time.new.to_i - start
+        new_count = destination.count(to_table, to_db) - original_count
+        source.forklift.logger.log("  ^ created #{new_count} new rows in #{delta}s")
+      end
+
+      def self.optimistic_pipe(source, from_table, destination, to_table, matcher=source.default_matcher, primary_key='id')
+        from_db = source.current_database 
+        to_db = destination.current_database 
+        if self.can_incremental_pipe?(from_db, from_table)
+          incremental_pipe(from_db, from_table, to_db, to_table, matcher, primary_key)
+        else
+          pipe(from_db, from_table, to_db, to_table)
+        end
+      end
+
+      def self.can_incremental_pipe?(conn, table, matcher=conn.default_matcher)
+        conn.columns(table, conn.current_database).include?(matcher)
+      end
+
+      ## When you are copying data to and from mysql 
+      ## An implamentation of "pipe" for remote databases
+      def self.mysql_optimistic_import(source, destination)
+        #TODO: allow passing in of matcher and primary_key
+        source.tables.each do |table|
+          if( source.columns(table).include?(source.default_matcher) && destination.tables.include?(table) )
+            since = destination.max_timestamp(table)
+            source.read_since(table, since){ |data| destination.write(data, table) }
+          else
+            destination.truncate table
+            source.read("select * from #{table}"){ |data| destination.write(data, table) }
+          end
+        end
+      end
+
+    end
+  end
+end