ETL 0.0.1 → 1.0.0.rc

data/Rakefile

@@ -1,2 +1,28 @@
 #!/usr/bin/env rake
 require "bundler/gem_tasks"
+begin
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:spec) do |t|
+    t.rspec_opts = '-b'
+  end
+
+  task default: :spec
+rescue LoadError
+  $stderr.puts "rspec not available, spec task not provided"
+end
+
+begin
+  require 'cane/rake_task'
+
+  desc "Run cane to check quality metrics"
+  Cane::RakeTask.new(:quality) do |cane|
+    cane.abc_max    = 10
+    cane.style_glob = "lib/**/*.rb"
+    cane.no_doc     = true
+  end
+
+  task :default => :quality
+rescue LoadError
+  warn "cane not available, quality task not provided."
+end

data/etl.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/etl/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Jeff Iacono"]
+  gem.email         = ["iacono@squareup.com"]
+  gem.description   = %q{Extract, Transform, and Load (ETL) ruby wrapper}
+  gem.summary       = %q{Extract, Transform, and Load (ETL) ruby wrapper. Supports basic and iterative ETL operations.}
+  gem.homepage      = "https://github.com/square/ETL"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "ETL"
+  gem.require_paths = ["lib"]
+  gem.version       = ETL::VERSION
+
+  gem.add_runtime_dependency "activesupport", [">= 3.2.3"]
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "cane"
+  gem.add_development_dependency "mysql2"
+  gem.add_development_dependency "rspec", [">= 2"]
+end

data/lib/etl.rb

@@ -0,0 +1,195 @@
+require 'etl/version'
+require 'etl/helpers'
+require 'logger'
+require 'date'
+require 'time'
+
+class ETL
+  include Helpers
+
+  attr_accessor :description
+  attr_accessor :connection
+  attr_reader   :logger
+
+  ORDERED_ETL_OPERATIONS = [
+    :ensure_destination,
+    :before_etl,
+    :etl,
+    :after_etl
+  ]
+
+  ITERATOR_OPERATIONS = [
+    :start,
+    :step,
+    :stop
+  ]
+
+  def initialize attributes = {}
+    attributes.keys.uniq.each do |attribute|
+      self.send "#{attribute}=", attributes[attribute]
+    end
+    default_logger! unless attributes.keys.include?(:logger)
+  end
+
+  def config &block
+    yield self if block_given?
+    self
+  end
+
+  def logger= logger
+    @logger = logger
+  end
+
+  # A little metaprogramming to consolidate the generation of our sql
+  # generating / querying methods. Note that we don't metaprogram the etl
+  # operation as it's a little more complex.
+  #
+  # This will produce methods of the form:
+  #
+  #   def [name] *args, &block
+  #     if block_given?
+  #       @[name] = block
+  #     else
+  #       @[name].call self, *args if @[name]
+  #     end
+  #   end
+  #
+  # for any given variable included in the method name's array
+  (ORDERED_ETL_OPERATIONS - [:etl]).each do |method|
+    define_method method do |*args, &block|
+      if block
+        instance_variable_set("@#{method}", block)
+      else
+        instance_variable_get("@#{method}").
+          call(self, *args) if instance_variable_get("@#{method}")
+      end
+    end
+  end
+
+  def etl *args, &block
+    if block_given?
+      @etl = block
+    else
+      if iterate?
+        if @etl
+          current = start
+          @etl.call self, cast(current), cast(current += step) while stop >= current
+        end
+      else
+        @etl.call self, *args if @etl
+      end
+    end
+  end
+
+  # A little more metaprogramming to consolidate the generation of
+  # our sql generating / querying methods.
+  #
+  # This will produce methods of the form:
+  #
+  #   def [method] *args, &block
+  #     if block
+  #       @_[method]_block = block
+  #     else
+  #       # cache block's result
+  #       if defined? @[method]
+  #         @[method]
+  #       else
+  #         @[method] = @_[method]_block.call(self, *args)
+  #       end
+  #     end
+  #   end
+  #
+  # for any given variable included in the method name's array
+  ITERATOR_OPERATIONS.each do |method|
+    define_method method do |*args, &block|
+      if block
+        instance_variable_set("@_#{method}_block", block)
+      else
+        if instance_variable_defined?("@#{method}")
+          instance_variable_get("@#{method}")
+        else
+          instance_variable_set("@#{method}",
+                                instance_variable_get("@_#{method}_block")
+                                  .call(self, *args))
+        end
+      end
+    end
+  end
+
+  def run options = {}
+    (ORDERED_ETL_OPERATIONS - [*options[:except]]).each do |method|
+      send method
+    end
+  end
+
+  def query sql
+    time_and_log(sql: sql) do
+      connection.query sql
+    end
+  end
+
+private
+
+  def iterate?
+    ITERATOR_OPERATIONS.all? do |method|
+      instance_variable_defined?("@_#{method}_block")
+    end
+  end
+
+  def default_logger!
+    @logger = default_logger
+  end
+
+  def logger?
+    !!@logger
+  end
+
+  def info data = {}
+    logger.info data.merge(emitter: self) if logger?
+  end
+
+  def debug data = {}
+    logger.debug data.merge(emitter: self) if logger?
+  end
+
+  def default_logger
+    ::Logger.new(STDOUT).tap do |logger|
+      logger.formatter = proc do |severity, datetime, progname, msg|
+        lead =  "[#{datetime}] #{severity} #{msg[:event_type]}"
+        desc =  "\"#{msg[:emitter].description || 'no description given'}\""
+        desc += " (object #{msg[:emitter].object_id})"
+
+        case msg[:event_type]
+        when :query_start
+          "#{lead} for #{desc}\n#{msg[:sql]}\n"
+        when :query_complete
+          "#{lead} for #{desc} runtime: #{msg[:runtime]}s\n"
+        else
+          "#{msg}"
+        end
+      end
+    end
+  end
+
+  def time_and_log data = {}, &block
+    start_runtime = Time.now
+    debug data.merge(event_type: :query_start)
+    retval = yield
+    info data.merge(event_type: :query_complete,
+                    runtime: Time.now - start_runtime)
+    retval
+  end
+
+  # NOTE: If you needed to handle more type data type casting you can add a
+  # case statement. If you need to be able to handle entirely different sets
+  # of casting depending on database engine, you can modify #cast to take a
+  # "type" arg and then determine which caster to route the arg through
+  def cast arg
+    case arg
+    when Date then arg.strftime("%Y-%m-%d")
+    when Time then arg.strftime("%Y-%m-%d %H:%M:%S")
+    else
+      arg
+    end
+  end
+end

data/lib/etl/helpers.rb

@@ -0,0 +1,57 @@
+class ETL
+  module Helpers
+    # max_for returns the max value for the passed in column as found in the
+    # specified database.table. If there is not currently a max, we use COALESCE
+    # and a default value. You can specify a :default_floor value or the method
+    # will try to derive it for you.
+    #
+    # Note: we try to detect if we want a date return type via the #datetype?
+    # check.
+    #
+    # If this is found we wrap the whole SELECT clause in a DATE so it is cast
+    # accordingly.
+    def max_for options = {}
+      database = options[:database]
+      table    = options[:table]
+      column   = options[:column]
+
+      default_value = options[:default_floor] ||
+                        default_floor_for(column)
+
+      if date? default_value
+        default_value = "DATE('#{default_value}')"
+        caster = ->(str) { "DATE(#{str})" }
+      end
+
+      max_sql_clause = "COALESCE(MAX(#{table}.#{column}), #{default_value})"
+      max_sql_clause = caster.(max_sql_clause) if caster
+
+      sql = <<-EOS
+        SELECT #{max_sql_clause} AS the_max
+        FROM #{database}.#{table}
+      EOS
+      sql += " WHERE #{options[:conditions]}" if options[:conditions]
+
+      query(sql).to_a.first['the_max']
+    end
+
+  private
+
+    def date? val
+      val =~ /^\d{4}-\d{1,2}-\d{1,2}( \d{2}:\d{2}:\d{2}( ((-|\+)\d+)| UTC)?)?$/
+    end
+
+    def default_floor_for column
+      case column
+      when /_at$/
+        return '1970-01-01'
+      when /_date$/
+        return '1970-01-01'
+      when /(^id$|_id$)/
+        return 0
+      else
+        raise ArgumentError, "could not determine a default for #{column}"
+      end
+    end
+  end
+end

data/lib/etl/version.rb

@@ -0,0 +1,3 @@
+class ETL
+  VERSION = "1.0.0.rc"
+end

data/spec/etl_spec.rb

@@ -0,0 +1,622 @@
+require 'mysql2'
+require 'active_support/time'
+require 'etl'
+
+def test_connection
+  Mysql2::Client.new host: 'localhost', username: 'root', database: 'etl_test'
+end
+
+def reset_test_env connection, &block
+  connection.query %[DROP DATABASE IF EXISTS etl_test]
+  connection.query %[CREATE DATABASE etl_test]
+  connection.query %[USE etl_test]
+
+  if block_given?
+    yield connection
+  else
+    connection.query %[
+      CREATE TABLE etl_source (
+          id INT NOT NULL
+        , name VARCHAR(10)
+        , amount INT(11) DEFAULT 0
+        , PRIMARY KEY (id))]
+
+    connection.query %[
+      INSERT INTO etl_test.etl_source (id, name, amount)
+      VALUES
+        (1, 'Jeff', 100),
+        (2, 'Ryan',  50),
+        (3, 'Jack',  75),
+        (4, 'Jeff',  10),
+        (5, 'Jack',  45),
+        (6, 'Nick', -90),
+        (7, 'Nick',  90)
+    ]
+  end
+end
+
+describe ETL do
+  let(:logger) { nil }
+
+  describe "#logger=" do
+    let(:etl) { described_class.new connection: stub }
+
+    it 'assigns' do
+      logger = stub
+      etl.logger = logger
+      etl.logger.should == logger
+    end
+  end
+
+  describe '#max_for' do
+    let(:connection) { test_connection }
+    let(:etl)        { described_class.new connection: connection, logger: logger }
+
+    before do
+      client = Mysql2::Client.new host: 'localhost', username: 'root'
+      client.query %[DROP DATABASE IF EXISTS etl_test]
+      client.query %[CREATE DATABASE etl_test]
+      client.query %[USE etl_test]
+      client.query %[
+        CREATE TABLE IF NOT EXISTS etl_source (
+            id INT(11) NOT NULL AUTO_INCREMENT
+          , name VARCHAR(10)
+          , amount INT(11) DEFAULT 0
+          , the_date DATE DEFAULT NULL
+          , the_null_date DATE DEFAULT NULL
+          , the_time_at DATETIME DEFAULT NULL
+          , the_null_time_at DATETIME DEFAULT NULL
+          , PRIMARY KEY (id))]
+
+      client.query %[
+        INSERT INTO etl_source (
+            name
+          , amount
+          , the_date
+          , the_null_date
+          , the_time_at
+          , the_null_time_at
+        ) VALUES
+            ('Jeff', 100, '2012-01-02', NULL, '2012-01-02 00:00:01', NULL)
+          , ('Ryan',  50, '2012-01-01', NULL, '2012-01-01 00:00:00', NULL)
+          , ('Jack',  75, '2012-01-01', NULL, '2012-01-01 00:00:00', NULL)
+          , ('Jeff',  10, '2012-01-01', NULL, '2012-01-01 00:00:00', NULL)
+          , ('Jack',  45, '2012-01-01', NULL, '2012-01-01 00:00:00', NULL)
+          , ('Nick', -90, '2012-01-01', NULL, '2012-01-01 00:00:00', NULL)
+          , ('Nick',  90, '2012-01-01', NULL, '2012-01-01 00:00:00', NULL)]
+
+      client.close
+    end
+
+    after { connection.close }
+
+    it "finds the max for dates" do
+      etl.max_for(database: :etl_test,
+                  table:    :etl_source,
+                  column:   :the_date).should == Date.parse('2012-01-02')
+    end
+
+    it "defaults to the beginning of time date when a max date cannot be found" do
+      etl.max_for(database: :etl_test,
+                  table:    :etl_source,
+                  column:   :the_null_date).should == Date.parse('1970-01-01')
+    end
+
+    it "defaults to the specified default floor when a max date cannot be found" do
+      etl.max_for(database:      :etl_test,
+                  table:         :etl_source,
+                  column:        :the_null_date,
+                  default_floor: '2011-01-01').should == Date.parse('2011-01-01')
+    end
+
+    it "finds the max for datetimes" do
+      etl.max_for(database: :etl_test,
+                  table:    :etl_source,
+                  column:   :the_time_at).should == Date.parse('2012-01-02')
+    end
+
+    it "defaults to the beginning of time when a max datetime cannot be found" do
+      etl.max_for(database: :etl_test,
+                  table:    :etl_source,
+                  column:   :the_null_time_at).should == Date.parse('1970-01-01 00:00:00')
+    end
+
+    it "defaults to the specified default floor when a max datetime cannot be found" do
+      etl.max_for(database:      :etl_test,
+                  table:         :etl_source,
+                  column:        :the_null_time_at,
+                  default_floor: '2011-01-01 00:00:00').should == Date.parse('2011-01-01 00:00:00')
+    end
+
+    it "raises an error if a non-standard column is supplied with no default floor" do
+      expect {
+        etl.max_for database: :etl_test,
+                    table:    :etl_source,
+                    column:   :amount
+      }.to raise_exception
+    end
+
+    it "finds the max for a non-standard column, using the default floor" do
+      etl.max_for(database:      :etl_test,
+                  table:         :etl_source,
+                  column:        :amount,
+                  default_floor: 0).should == 100
+    end
+  end
+
+  describe '#run' do
+    let(:connection) { test_connection }
+    let(:etl)        { described_class.new connection: connection, logger: logger }
+
+    before do
+      client = Mysql2::Client.new host: 'localhost', username: 'root'
+      client.query %[DROP DATABASE IF EXISTS etl_test]
+      client.query %[CREATE DATABASE etl_test]
+      client.query %[USE etl_test]
+      client.query %[
+        CREATE TABLE IF NOT EXISTS etl_source (
+            id INT(11) NOT NULL AUTO_INCREMENT
+          , name VARCHAR(10)
+          , amount INT(11) DEFAULT 0
+          , PRIMARY KEY (id))]
+
+      client.query %[
+        INSERT INTO etl_source (name, amount)
+        VALUES
+          ('Jeff',  100),
+          ('Ryan',  50),
+          ('Jack',  75),
+          ('Jeff',  10),
+          ('Jack',  45),
+          ('Nick', -90),
+          ('Nick',  90)]
+
+      client.close
+    end
+
+    it "executes the specified sql in the appropriate order" do
+      etl.ensure_destination do |etl|
+        etl.query %[
+          CREATE TABLE IF NOT EXISTS etl_destination (
+            name VARCHAR(10)
+          , total_amount INT(11) DEFAULT 0
+          , PRIMARY KEY (name))]
+      end
+
+      etl.before_etl do |etl|
+        etl.query "DELETE FROM etl_source WHERE amount < 0"
+      end
+
+      etl.etl do |etl|
+        etl.query %[
+          REPLACE INTO etl_destination
+          SELECT name, SUM(amount) FROM etl_source
+          GROUP BY name]
+      end
+
+      etl.after_etl do |etl|
+        etl.query %[
+          UPDATE etl_destination
+          SET name = CONCAT("SUPER ", name)
+          WHERE total_amount > 115]
+      end
+
+      etl.run
+
+      connection
+        .query("SELECT * FROM etl_destination ORDER BY total_amount DESC")
+        .to_a
+        .should == [
+          {'name' => 'SUPER Jack', 'total_amount' => 120},
+          {'name' => 'Jeff',       'total_amount' => 110},
+          {'name' => 'Nick',       'total_amount' => 90},
+          {'name' => 'Ryan',       'total_amount' => 50}]
+    end
+  end
+
+  describe '#run with operations specified for exclusion' do
+    let(:connection) { stub }
+    let(:etl)        { described_class.new connection: connection, logger: logger }
+
+    it "does not call the specified method" do
+      etl.ensure_destination {}
+      etl.should_not_receive(:ensure_destination)
+      etl.run except: :ensure_destination
+    end
+  end
+
+  context "with iteration" do
+    describe '#run over full table' do
+      let(:connection) { test_connection }
+      let(:etl)        { described_class.new connection: connection, logger: logger }
+
+      before { reset_test_env connection }
+      after  { connection.close }
+
+      it "executes the specified sql in the appropriate order and ETLs properly" do
+        etl.ensure_destination do |etl|
+          etl.query %[
+            CREATE TABLE etl_destination (
+                id INT NOT NULL
+              , name VARCHAR(10)
+              , amount INT(11) DEFAULT 0
+              , PRIMARY KEY (id))]
+        end
+
+        etl.before_etl do |etl|
+          etl.query "DELETE FROM etl_source WHERE amount < 0"
+        end
+
+        etl.start do |etl|
+          etl.query(
+            "SELECT COALESCE(MAX(id), 0) AS the_start FROM etl_destination"
+          ).to_a.first['the_start']
+        end
+
+        etl.step do
+          1
+        end
+
+        etl.stop do |etl|
+          etl.query(
+            "SELECT MAX(id) AS the_stop FROM etl_source"
+          ).to_a.first['the_stop']
+        end
+
+        etl.etl do |etl, lbound, ubound|
+          etl.query %[
+            REPLACE INTO etl_destination
+            SELECT id, name, amount FROM etl_source s
+            WHERE s.id >= #{lbound}
+              AND s.id <  #{ubound}]
+        end
+
+        etl.after_etl do |etl|
+          etl.query %[
+            UPDATE etl_destination
+            SET name = CONCAT("SUPER ", name)
+            WHERE id <= 1]
+        end
+
+        etl.run
+
+        connection
+          .query("SELECT * FROM etl_destination ORDER BY id ASC")
+          .to_a
+          .should == [
+            {'id' => 1, 'name' => 'SUPER Jeff', 'amount' => 100},
+            {'id' => 2, 'name' => 'Ryan',       'amount' => 50},
+            {'id' => 3, 'name' => 'Jack',       'amount' => 75},
+            {'id' => 4, 'name' => 'Jeff',       'amount' => 10},
+            {'id' => 5, 'name' => 'Jack',       'amount' => 45},
+            {'id' => 7, 'name' => 'Nick',       'amount' => 90}]
+      end
+    end
+
+    describe '#run over part of table' do
+      let(:connection) { test_connection }
+      let(:etl)        { described_class.new connection: connection, logger: logger }
+
+      before { reset_test_env connection }
+      after  { connection.close }
+
+      it "executes the specified sql in the appropriate order and ETLs properly" do
+        etl.ensure_destination do |etl|
+          etl.query %[
+            CREATE TABLE etl_destination (
+                id INT NOT NULL
+              , name VARCHAR(10)
+              , amount INT(11) DEFAULT 0
+              , PRIMARY KEY (id))]
+        end
+
+        etl.before_etl do |etl|
+          etl.query "DELETE FROM etl_source WHERE amount < 0"
+        end
+
+        etl.start do
+          4
+        end
+
+        etl.step do
+          1
+        end
+
+        etl.stop do |etl|
+          etl.query(
+            "SELECT MAX(id) AS the_stop FROM etl_source"
+          ).to_a.first['the_stop']
+        end
+
+        etl.etl do |etl, lbound, ubound|
+          etl.query %[
+            REPLACE INTO etl_destination
+            SELECT id, name, amount FROM etl_source s
+            WHERE s.id >= #{lbound}
+              AND s.id <  #{ubound}]
+        end
+
+        etl.run
+
+        connection
+          .query("SELECT * FROM etl_destination ORDER BY id ASC")
+          .to_a.should == [
+            {'id' => 4, 'name' => 'Jeff', 'amount' => 10},
+            {'id' => 5, 'name' => 'Jack', 'amount' => 45},
+            {'id' => 7, 'name' => 'Nick', 'amount' => 90}]
+      end
+    end
+
+    describe "#run over gappy data" do
+      let(:connection) { test_connection }
+      let(:etl)        { described_class.new connection: connection, logger: logger }
+
+      before do
+        reset_test_env(connection) do |connection|
+          connection.query %[
+            CREATE TABLE etl_source (
+                id INT NOT NULL
+              , name VARCHAR(10)
+              , amount INT(11) DEFAULT 0
+              , PRIMARY KEY (id))]
+
+          connection.query %[
+            INSERT INTO etl_source (id, name, amount)
+            VALUES
+              (1,  'Jeff',  100),
+              (2,  'Ryan',  50),
+              (13, 'Jack',  75),
+              (14, 'Jeff',  10),
+              (15, 'Jack',  45),
+              (16, 'Nick', -90),
+              (17, 'Nick',  90)]
+        end
+      end
+
+      after { connection.close }
+
+      it "executes the specified sql in the appropriate order without getting stuck" do
+        etl.ensure_destination do |etl|
+          etl.query %[
+            CREATE TABLE etl_destination (
+                id INT NOT NULL
+              , name VARCHAR(10)
+              , amount INT(11) DEFAULT 0
+              , PRIMARY KEY (id))]
+        end
+
+        etl.before_etl do |etl|
+          etl.query "DELETE FROM etl_source WHERE amount < 0"
+        end
+
+        etl.start do |etl|
+          1
+        end
+
+        etl.step do
+          1
+        end
+
+        etl.stop do |etl|
+          etl.query(
+            "SELECT MAX(id) AS the_stop FROM etl_source"
+          ).to_a.first['the_stop']
+        end
+
+        etl.etl do |etl, lbound, ubound|
+          etl.query %[
+            REPLACE INTO etl_destination
+            SELECT
+                id
+              , name
+              , amount
+            FROM etl_source s
+            WHERE s.id >= #{lbound}
+              AND s.id <  #{ubound}]
+        end
+
+        etl.run
+
+        connection
+          .query("SELECT * FROM etl_destination ORDER BY id ASC")
+          .to_a
+          .should == [
+            {'id' => 1,  'name' => 'Jeff', 'amount' => 100},
+            {'id' => 2,  'name' => 'Ryan', 'amount' => 50},
+            {'id' => 13, 'name' => 'Jack', 'amount' => 75},
+            {'id' => 14, 'name' => 'Jeff', 'amount' => 10},
+            {'id' => 15, 'name' => 'Jack', 'amount' => 45},
+            {'id' => 17, 'name' => 'Nick', 'amount' => 90}]
+      end
+    end
+
+    describe "#run over date data" do
+      let(:connection) { test_connection }
+      let(:etl)        { described_class.new connection: connection, logger: logger }
+
+      before do
+        reset_test_env(connection) do |connection|
+          connection.query %[
+            CREATE TABLE etl_source (
+                the_date DATE NOT NULL
+              , name VARCHAR(10)
+              , amount INT(11) DEFAULT 0)]
+
+          connection.query %[
+            INSERT INTO etl_source (the_date, name, amount)
+            VALUES
+              ('2012-01-01', 'Jeff', 100),
+              ('2012-01-01', 'Ryan', 50),
+              ('2012-01-01', 'Jack', 75),
+              ('2012-01-01', 'Jeff', 10),
+              ('2012-01-02', 'Jack', 45),
+              ('2012-01-02', 'Nick', -90),
+              ('2012-01-02', 'Nick', 90)]
+        end
+      end
+
+      after { connection.close }
+
+      it "executes the specified sql in the appropriate order and ETLs properly" do
+        etl.ensure_destination do |etl|
+          etl.query %[
+            CREATE TABLE etl_destination (
+                the_date DATE NOT NULL
+              , name VARCHAR(10)
+              , total_amount INT(11) DEFAULT 0
+              , PRIMARY KEY (the_date, name))]
+        end
+
+        etl.before_etl do |etl|
+          etl.query "DELETE FROM etl_source WHERE amount < 0"
+        end
+
+        etl.start do |etl|
+          etl.query(%[
+            SELECT COALESCE(MAX(the_date), DATE('2012-01-01')) AS the_start
+            FROM etl_destination
+          ]).to_a.first['the_start']
+        end
+
+        etl.step do
+          1.day
+        end
+
+        etl.stop do |etl|
+          etl.query(
+            "SELECT MAX(the_date) AS the_stop FROM etl_source"
+          ).to_a.first['the_stop']
+        end
+
+        etl.etl do |etl, lbound, ubound|
+          etl.query %[
+            REPLACE INTO etl_destination
+            SELECT
+                the_date
+              , name
+              , SUM(amount) AS total_amount
+            FROM etl_source s
+            WHERE s.the_date >= '#{lbound}'
+              AND s.the_date <  '#{ubound}'
+            GROUP BY
+                the_date
+              , name]
+        end
+
+        etl.run
+
+        connection
+          .query(%[
+            SELECT
+                the_date
+              , name
+              , total_amount
+            FROM
+              etl_destination
+            ORDER BY
+                the_date ASC
+              , name ASC
+          ]).to_a
+            .should == [
+              {'the_date' => Date.parse('2012-01-01'), 'name' => 'Jack', 'total_amount' => 75},
+              {'the_date' => Date.parse('2012-01-01'), 'name' => 'Jeff', 'total_amount' => 110},
+              {'the_date' => Date.parse('2012-01-01'), 'name' => 'Ryan', 'total_amount' => 50},
+              {'the_date' => Date.parse('2012-01-02'), 'name' => 'Jack', 'total_amount' => 45},
+              {'the_date' => Date.parse('2012-01-02'), 'name' => 'Nick', 'total_amount' => 90}]
+      end
+    end
+
+    describe "#run over datetime data" do
+      let(:connection) { test_connection }
+      let(:etl)        { described_class.new connection: connection, logger: logger }
+
+      before do
+        reset_test_env(connection) do |connection|
+          connection.query %[
+            CREATE TABLE etl_source (
+                the_datetime DATETIME NOT NULL
+              , name VARCHAR(10)
+              , amount INT(11) DEFAULT 0)]
+
+          connection.query %[
+            INSERT INTO etl_source (the_datetime, name, amount)
+            VALUES
+              ('2011-12-31 23:59:59', 'Jeff', 100),
+              ('2012-01-01 00:01:00', 'Ryan', 50),
+              ('2012-01-01 00:01:01', 'Jack', 75),
+              ('2012-01-01 00:01:02', 'Jeff', 10),
+              ('2012-01-02 00:02:00', 'Jack', 45),
+              ('2012-01-02 00:02:01', 'Nick', -90),
+              ('2012-01-02 00:02:02', 'Nick', 90)]
+        end
+      end
+
+      after { connection.close }
+
+      it "executes the specified sql in the appropriate order and ETLs properly" do
+        etl.ensure_destination do |etl|
+          etl.query %[
+            CREATE TABLE etl_destination (
+                the_datetime DATETIME NOT NULL
+              , name VARCHAR(10)
+              , amount INT(11) DEFAULT 0
+              , PRIMARY KEY (the_datetime, name))]
+        end
+
+        etl.before_etl do |etl|
+          etl.query "DELETE FROM etl_source WHERE amount < 0"
+        end
+
+        etl.start do |etl|
+          etl.query(%[
+            SELECT CAST(COALESCE(MAX(the_datetime), '2012-01-01 00:00:00') AS DATETIME) AS the_start
+            FROM etl_destination
+          ]).to_a.first['the_start']
+        end
+
+        etl.step do
+          1.minute
+        end
+
+        etl.stop do |etl|
+          etl.query(
+            "SELECT MAX(the_datetime) AS the_stop FROM etl_source"
+          ).to_a.first['the_stop']
+        end
+
+        etl.etl do |etl, lbound, ubound|
+          etl.query %[
+            REPLACE INTO etl_destination
+            SELECT
+                the_datetime
+              , name
+              , amount
+            FROM etl_source s
+            WHERE s.the_datetime >= '#{lbound}'
+              AND s.the_datetime <  '#{ubound}']
+        end
+
+        etl.run
+
+        connection
+          .query(%[
+            SELECT
+                the_datetime
+              , name
+              , amount
+            FROM
+              etl_destination
+            ORDER BY
+                the_datetime ASC
+              , name ASC
+          ]).to_a
+            .should == [
+              {'the_datetime' => Time.parse('2012-01-01 00:01:00'), 'name' => 'Ryan', 'amount' => 50},
+              {'the_datetime' => Time.parse('2012-01-01 00:01:01'), 'name' => 'Jack', 'amount' => 75},
+              {'the_datetime' => Time.parse('2012-01-01 00:01:02'), 'name' => 'Jeff', 'amount' => 10},
+              {'the_datetime' => Time.parse('2012-01-02 00:02:00'), 'name' => 'Jack', 'amount' => 45},
+              {'the_datetime' => Time.parse('2012-01-02 00:02:02'), 'name' => 'Nick', 'amount' => 90}]
+      end
+    end
+  end
+end