coupler 0.0.1-java

data/test/integration/test_import.rb

@@ -0,0 +1,78 @@
+require 'helper'
+
+class TestImport < Coupler::Test::IntegrationTest
+
+  test "import!" do
+    project = Project.create(:name => "foo")
+    import = Import.create(:data => fixture_file_upload("people.csv"), :project => project)
+    import.import!
+    assert_not_nil import.occurred_at
+
+    project.local_database do |db|
+      name = :"import_#{import.id}"
+      assert db.tables.include?(name)
+      schema = db.schema(name)
+      assert_equal [:integer, true], schema.assoc(:id)[1].values_at(:type, :primary_key)
+      assert_equal :string, schema.assoc(:first_name)[1][:type]
+      assert_equal :string, schema.assoc(:last_name)[1][:type]
+      assert_equal :integer, schema.assoc(:age)[1][:type]
+
+      ds = db[name]
+      assert_equal 50, ds.count
+    end
+  end
+
+  test "flags duplicate primary keys" do
+    tempfile = Tempfile.new('coupler-import')
+    tempfile.write("id,foo,bar\n1,abc,def\n2,ghi,jkl\n2,mno,pqr")
+    tempfile.close
+
+    project = Project.create(:name => "foo")
+    import = Import.create(:data => file_upload(tempfile.path), :project => project)
+
+    assert !import.import!
+    assert import.has_duplicate_keys
+
+    project.local_database do |db|
+      ds = db[:"import_#{import.id}"]
+      assert ds.filter(:id => 2).select_map(:dup_key_count).all?
+    end
+  end
+
+  test "repair duplicate keys" do
+    project = Project.create(:name => "foo")
+    import = Import.create(:data => fixture_file_upload('duplicate-keys.csv'), :project => project)
+    import.import!
+
+    import.repair_duplicate_keys!(nil)
+    import.dataset do |ds|
+      assert !ds.columns!.include?(:dup_key_count)
+      assert_equal 1, ds.filter(:id => 2).count
+      assert_equal 1, ds.filter(:id => 4).count
+    end
+    project.local_database do |db|
+      assert db.schema(:"import_#{import.id}").assoc(:id)[1][:primary_key]
+    end
+  end
+
+  test "repair duplicate keys with deletions" do
+    tempfile = Tempfile.new('coupler-import')
+    tempfile.write("id,foo,bar\n1,2,3\n1,4,5\n1,6,7\n123,456,789\n")
+    tempfile.close
+
+    project = Project.create(:name => "foo")
+    import = Import.create(:data => file_upload(tempfile.path), :project => project)
+    import.import!
+
+    import.repair_duplicate_keys!({"1" => ["1"]})
+    import.dataset do |ds|
+      assert !ds.columns!.include?(:dup_key_count)
+      assert_equal 1, ds.filter(:id => 1).count
+      assert_equal 1, ds.filter(:id => 124).count
+      assert_equal 0, ds.filter(:id => 125).count
+    end
+    project.local_database do |db|
+      assert db.schema(:"import_#{import.id}").assoc(:id)[1][:primary_key]
+    end
+  end
+end

data/test/integration/test_running_scenarios.rb

@@ -0,0 +1,379 @@
+require 'helper'
+
+class TestRunningScenarios < Coupler::Test::IntegrationTest
+
+  def self.startup
+    super
+    Connection.delete
+    each_adapter do |adapter, config|
+      conn = new_connection(adapter, :name => "#{adapter} connection").save!
+      conn.database do |db|
+        tables = db.tables
+        db.drop_table(:basic_self_linkage)  if tables.include?(:basic_self_linkage)
+        db.drop_table(:basic_cross_linkage) if tables.include?(:basic_cross_linkage)
+        TableMaker.new(db, :basic_self_linkage, <<-EOF)
+          +-------------+-------------+
+          | id(Integer) | foo(String) |
+          +=============+=============+
+          | 1           | foo         |
+          | 2           | foo         |
+          | 3           | foo         |
+          | 4           | foo         |
+          | 5           | foo         |
+          | 6           | bar         |
+          | 7           | bar         |
+          | 8           | bar         |
+          | 9           | bar         |
+          | 10          | bar         |
+          | 11          | baz         |
+          | 12          | baz         |
+          | 13          | baz         |
+          | 14          | baz         |
+          | 15          | baz         |
+          | 16          | abc         |
+          | 17          | def         |
+          | 18          | ghi         |
+          | 19          | jkl         |
+          | 20          | mno         |
+          +-------------+-------------+
+        EOF
+
+        TableMaker.new(db, :basic_cross_linkage, <<-EOF)
+          +-------------+-------------+-------------+
+          | id(Integer) | foo(String) | bar(String) |
+          +=============+=============+=============+
+          | 18          | foo         |             |
+          | 16          | foo         |             |
+          | 17          | foo         |             |
+          | 15          |             | foo         |
+          | 13          |             | foo         |
+          | 10          |             | foo         |
+          | 19          |             | foo         |
+          | 25          | bar         |             |
+          | 23          | bar         |             |
+          | 20          | bar         |             |
+          | 29          | bar         |             |
+          | 28          |             | bar         |
+          | 26          |             | bar         |
+          | 27          |             | bar         |
+          | 38          | baz         |             |
+          | 35          |             | baz         |
+          | 33          |             | baz         |
+          | 30          |             | baz         |
+          | 39          |             | baz         |
+          | 45          | quux        |             |
+          | 43          | quux        |             |
+          | 40          | quux        |             |
+          | 49          | quux        |             |
+          | 48          |             | quux        |
+          | 55          | ditto       | ditto       |
+          | 53          | ditto       | ditto       |
+          +-------------+-------------+-------------+
+        EOF
+
+        db.create_table!(:resource_1) do
+          primary_key :id
+          String :ssn, :index => true
+          String :dob, :index => true
+          Integer :foo, :index => true
+          Integer :bar, :index => true
+          Integer :age, :index => true
+          Integer :height, :index => true
+          index [:id, :ssn]
+          index [:id, :ssn, :dob]
+          index [:id, :foo, :bar]
+          index [:id, :age]
+        end
+        rows = Array.new(11000) do |i|
+          [
+            i < 10500 ? "1234567%02d"  % (i / 350) : "9876%05d" % i,
+            i < 10500 ? "2000-01-%02d" % (i / 525) : nil,
+            i < 1000 ? ((x = i / 125) == 2 ? 123 : x) : nil,
+            i < 1000 ? i % 10 : nil,
+            i % 20 + 25,
+            i % 50 + 125,
+          ]
+        end
+        db[:resource_1].import([:ssn, :dob, :foo, :bar, :age, :height], rows)
+
+        db.create_table!(:resource_2) do
+          primary_key :id
+          String :SocialSecurityNumber
+          index [:id, :SocialSecurityNumber]
+        end
+        rows = Array.new(21000) do |i|
+          [
+            i < 10500 ? "1234567%02d" % (i % 30) : "9876%05d" % i,
+          ]
+        end
+        db[:resource_2].import([:SocialSecurityNumber], rows)
+      end
+    end
+  end
+
+  #def self.shutdown
+    #each_adapter do |adapter, config|
+      #conn = Connection[:name => "#{adapter} connection"]
+      #conn.database do |db|
+        #db.drop_table(:basic_cross_linkage)
+        #db.drop_table(:basic_self_linkage)
+      #end
+    #end
+    #super
+  #end
+
+  def setup
+    super
+    @project = Project.create(:name => "foo")
+  end
+
+  each_adapter do |adapter, _|
+    adapter_test(adapter, "default csv export self linkage") do
+      conn = new_connection(adapter, :name => :"#{adapter} connection").save!
+      resource = Resource.create(:name => 'foo', :table_name => 'basic_self_linkage', :project => @project, :connection => conn)
+      scenario = Scenario.create(:name => 'foo', :resource_1 => resource, :project => @project)
+      foo_field = resource.fields_dataset[:name => 'foo']
+      matcher = Matcher.create({
+        :scenario => scenario,
+        :comparisons_attributes => [{
+          'lhs_type' => 'field', 'raw_lhs_value' => foo_field.id, 'lhs_which' => 1,
+          'rhs_type' => 'field', 'raw_rhs_value' => foo_field.id, 'rhs_which' => 2,
+          'operator' => 'equals'
+        }]
+      })
+      scenario.run!
+      assert_not_nil scenario.last_run_at
+      assert_equal 1, scenario.run_count
+      assert_equal 1, scenario.results_dataset.count
+
+      result = scenario.results_dataset.first
+      csv = result.to_csv
+
+      # FIXME: add some actual csv tests you lazy bastard
+    end
+
+    adapter_test(adapter, "default csv export cross linkage") do
+      conn = new_connection(adapter, :name => :"#{adapter} connection").save!
+      resource = Resource.create(:name => 'foo', :table_name => 'basic_cross_linkage', :project => @project, :connection => conn)
+      scenario = Scenario.create(:name => 'foo', :resource_1 => resource, :project => @project)
+      foo_field = resource.fields_dataset[:name => 'foo']
+      bar_field = resource.fields_dataset[:name => 'bar']
+      matcher = Matcher.create({
+        :scenario => scenario,
+        :comparisons_attributes => [{
+          'lhs_type' => 'field', 'raw_lhs_value' => foo_field.id, 'lhs_which' => 1,
+          'rhs_type' => 'field', 'raw_rhs_value' => bar_field.id, 'rhs_which' => 2,
+          'operator' => 'equals'
+        }]
+      })
+      scenario.run!
+      result = scenario.results_dataset.first
+      csv = result.to_csv
+
+      # FIXME: add some actual csv tests you lazy bastard
+    end
+
+    adapter_test(adapter, "self linkage with one comparison") do
+      conn = new_connection(adapter, :name => :"#{adapter} connection").save!
+      resource = Resource.create(:name => 'resource_1', :table_name => 'resource_1', :project => @project, :connection => conn)
+      scenario = Scenario.create(:name => 'foo', :resource_1 => resource, :project => @project)
+      field = resource.fields_dataset[:name => 'ssn']
+      matcher = Matcher.create({
+        :scenario => scenario, :comparisons_attributes => [{
+          'lhs_type' => 'field', 'raw_lhs_value' => field.id, 'lhs_which' => 1,
+          'rhs_type' => 'field', 'raw_rhs_value' => field.id, 'rhs_which' => 2,
+          'operator' => 'equals'
+        }]
+      })
+      scenario.run!
+
+      groups = {}
+      scenario.local_database do |db|
+        assert db.tables.include?(:groups_records_1)
+        ds = db[:groups_records_1]
+        assert_equal 10500, ds.count
+        counts = ds.group_and_count(:group_id).all
+        assert_equal 30, counts.length
+        assert counts.all? { |g| g[:count] == 350 }
+        assert ds.group_and_count(:record_id).all? { |r| r[:count] == 1 }
+        ds.each do |row|
+          record_id = groups[row[:group_id]]
+          if record_id
+            assert_equal (record_id - 1) / 350, (row[:record_id].to_i - 1) / 350, "Record #{row[:record_id]} should not have been in the same group as Record #{record_id}."
+          else
+            groups[row[:group_id]] = row[:record_id].to_i
+          end
+        end
+      end
+    end
+
+    adapter_test(adapter, "self linkage with two comparisons") do
+      conn = new_connection(adapter, :name => :"#{adapter} connection").save!
+      resource = Resource.create(:name => 'resource_1', :table_name => 'resource_1', :project => @project, :connection => conn)
+      scenario = Scenario.create(:name => 'foo', :resource_1 => resource, :project => @project)
+      field_1 = resource.fields_dataset[:name => 'ssn']
+      field_2 = resource.fields_dataset[:name => 'dob']
+      matcher = Matcher.create({
+        :scenario => scenario, :comparisons_attributes => [
+          {
+            'lhs_type' => 'field', 'raw_lhs_value' => field_1.id, 'lhs_which' => 1,
+            'rhs_type' => 'field', 'raw_rhs_value' => field_1.id, 'rhs_which' => 2,
+            'operator' => 'equals'
+          },
+          {
+            'lhs_type' => 'field', 'raw_lhs_value' => field_2.id, 'lhs_which' => 1,
+            'rhs_type' => 'field', 'raw_rhs_value' => field_2.id, 'rhs_which' => 2,
+            'operator' => 'equals'
+          },
+        ]
+      })
+      scenario.run!
+
+      groups = {}
+      scenario.local_database do |db|
+        assert db.tables.include?(:groups_records_1)
+        ds = db[:groups_records_1]
+        assert_equal 10500, ds.count
+
+        counts = ds.group_and_count(:group_id)
+        assert_equal 20, counts.having(:count => 175).count
+        assert_equal 20, counts.having(:count => 350).count
+        assert ds.group_and_count(:record_id).all? { |r| r[:count] == 1 }
+        ds.each do |row|
+          record_id = groups[row[:group_id]]
+          if record_id
+            assert_equal (record_id - 1) / 350, (row[:record_id].to_i - 1) / 350, "Record #{row[:record_id]} should not have been in the same group as Record #{record_id}."
+            assert_equal (record_id - 1) / 525, (row[:record_id].to_i - 1) / 525, "Record #{row[:record_id]} should not have been in the same group as Record #{record_id}."
+          else
+            groups[row[:group_id]] = row[:record_id].to_i
+          end
+        end
+      end
+    end
+
+    adapter_test(adapter, "self linkage with cross match") do
+      conn = new_connection(adapter, :name => :"#{adapter} connection").save!
+      resource = Resource.create(:name => 'resource_1', :table_name => 'resource_1', :project => @project, :connection => conn)
+      scenario = Scenario.create(:name => 'foo', :resource_1 => resource, :project => @project)
+      field_1 = resource.fields_dataset[:name => 'foo']
+      field_2 = resource.fields_dataset[:name => 'bar']
+      matcher = Matcher.create({
+        :scenario => scenario, :comparisons_attributes => [
+          {
+            'lhs_type' => 'field', 'raw_lhs_value' => field_1.id, 'lhs_which' => 1,
+            'rhs_type' => 'field', 'raw_rhs_value' => field_2.id, 'rhs_which' => 2,
+            'operator' => 'equals'
+          },
+        ]
+      })
+      scenario.run!
+
+      groups = {}
+      scenario.local_database do |db|
+        assert db.tables.include?(:groups_records_1)
+        join_ds = db[:groups_records_1]
+        # Breakdown of groups_records_1
+        # - Values that should match each other: 0, 1, 3, 4, 5, 6, 7
+        # - For each value, there are 125 records that should match in foo
+        #   Subtotal: 875
+        # - For each value, there are 100 records that should match in bar
+        #   Subtotal: 700
+        # * Expected Total: 1575
+        assert_equal 1575, join_ds.count
+
+        assert db.tables.include?(:groups_1)
+        group_ds = db[:groups_1]
+        assert_equal 7, group_ds.count
+        group_ds.each do |group_row|
+          assert_equal 125, group_row[:"resource_1_count"], "Row counts didn't match"
+          assert_equal 100, group_row[:"resource_2_count"], "Row counts didn't match"
+        end
+
+        assert_equal 0, join_ds.group_and_count(:group_id).having(:count => 1).count
+      end
+    end
+
+    adapter_test(adapter, "self linkage with blocking") do
+      conn = new_connection(adapter, :name => :"#{adapter} connection").save!
+      resource = Resource.create(:name => 'resource_1', :table_name => 'resource_1', :project => @project, :connection => conn)
+      scenario = Scenario.create(:name => 'foo', :resource_1 => resource, :project => @project)
+      field_1 = resource.fields_dataset[:name => 'age']
+      field_2 = resource.fields_dataset[:name => 'height']
+      matcher = Matcher.create({
+        :scenario => scenario, :comparisons_attributes => [
+          {
+            'lhs_type' => 'field', 'raw_lhs_value' => field_1.id, 'lhs_which' => 1,
+            'rhs_type' => 'field', 'raw_rhs_value' => field_1.id, 'rhs_which' => 2,
+            'operator' => 'equals'
+          },
+          {
+            'lhs_type' => 'field', 'raw_lhs_value' => field_1.id, 'lhs_which' => 1,
+            'rhs_type' => 'integer', 'raw_rhs_value' => 30,
+            'operator' => 'greater_than'
+          },
+          {
+            'lhs_type' => 'field', 'raw_lhs_value' => field_2.id, 'lhs_which' => 1,
+            'rhs_type' => 'integer', 'raw_rhs_value' => 150,
+            'operator' => 'greater_than'
+          },
+        ]
+      })
+      scenario.run!
+
+      groups = {}
+      scenario.local_database do |db|
+        assert db.tables.include?(:groups_records_1)
+        ds = db[:groups_records_1]
+        assert ds.group_and_count(:record_id).all? { |r| r[:count] == 1 }
+        ds.each do |row|
+          index = row[:record_id].to_i - 1
+          assert index % 20 > 5,  "#{row[:record_id]}'s age is too small"
+          assert index % 50 > 25, "#{row[:record_id]}'s height is too small"
+
+          record_id = groups[row[:group_id]]
+          if record_id
+            assert_equal (record_id - 1) % 20 + 25, (row[:record_id].to_i - 1) % 20 + 25, "Record #{row[:record_id]} should not have been in the same group as Record #{record_id}."
+          else
+            groups[row[:group_id]] = row[:record_id].to_i
+          end
+        end
+      end
+    end
+
+    adapter_test(adapter, "dual linkage with one comparison") do
+      conn = new_connection(adapter, :name => :"#{adapter} connection").save!
+      resource_1 = Resource.create(:name => 'resource_1', :table_name => 'resource_1', :project => @project, :connection => conn)
+      resource_2 = Resource.create(:name => 'resource_2', :table_name => 'resource_2', :project => @project, :connection => conn)
+      scenario = Scenario.create(:name => 'foo', :resource_1 => resource_1, :resource_2 => resource_2, :project => @project)
+      field_1 = resource_1.fields_dataset[:name => 'ssn']
+      field_2 = resource_2.fields_dataset[:name => 'SocialSecurityNumber']
+      assert field_1, "ssn field couldn't be found: #{resource_1.fields_dataset.collect(&:name).inspect}"
+      assert field_2, "socialsecuritynumber field couldn't be found: #{resource_2.fields_dataset.collect(&:name).inspect}"
+      matcher = Matcher.create({
+        :scenario => scenario, :comparisons_attributes => [{
+          'lhs_type' => 'field', 'raw_lhs_value' => field_1.id, 'lhs_which' => 1,
+          'rhs_type' => 'field', 'raw_rhs_value' => field_2.id, 'rhs_which' => 2,
+          'operator' => 'equals'
+        }]
+      })
+      scenario.run!
+
+      groups = {}
+      scenario.local_database do |db|
+        assert db.tables.include?(:groups_records_1)
+        ds = db[:groups_records_1]
+        assert_equal 22000, ds.count
+
+        counts = ds.group_and_count(:group_id).all
+        assert_equal 530, counts.length
+        counts = counts.inject({}) { |h, r| h[r[:count]] ||= 0; h[r[:count]] += 1; h }
+        assert_equal 30, counts[700]
+        assert_equal 500, counts[2]
+        assert ds.group_and_count(:record_id, :which).all? { |r| r[:count] == 1 }
+
+        ds = db[:groups_1]
+        assert_equal 530, ds.count
+      end
+    end
+  end
+end

data/test/integration/test_transformation.rb

@@ -0,0 +1,56 @@
+require 'helper'
+
+class TestTransformation < Coupler::Test::IntegrationTest
+  def self.startup
+    super
+    Connection.delete
+    each_adapter do |adapter, config|
+      data = Array.new(50) { [Forgery(:name).first_name, Forgery(:name).last_name] }
+      conn = new_connection(adapter, :name => "#{adapter} connection").save!
+      conn.database do |db|
+        db.create_table!(:test_data) do
+          primary_key :id
+          String :first_name
+          String :last_name
+        end
+        db[:test_data].import([:first_name, :last_name], data)
+      end
+    end
+  end
+
+  def adapter_setup(adapter)
+    @connection = new_connection(adapter, :name => "#{adapter} connection").save!
+    @project = Project.create(:name => 'foo')
+    @resource = Resource.create(:name => 'test', :connection => @connection, :table_name => 'test_data', :project => @project)
+  end
+
+  each_adapter do |adapter, _|
+    adapter_test(adapter, "uses local db type to determine field type") do
+      adapter_setup(adapter)
+      string_to_int = Transformer.create(:name => 'string_to_int', :allowed_types => %w{string}, :code => "value.length", :result_type => "integer")
+      int_to_string = Transformer.create(:name => 'int_to_string', :allowed_types => %w{integer}, :code => "value.to_s", :result_type => "string")
+      field = @resource.fields_dataset[:name => 'first_name']
+      xformation_1 = Transformation.create(:transformer => string_to_int, :source_field => field, :resource => @resource)
+      xformation_2 = Transformation.new(:transformer => int_to_string, :source_field => field, :resource => @resource)
+      assert xformation_2.valid?, xformation_2.errors.full_messages.join("; ")
+    end
+
+    adapter_test(adapter, "accepts nested attributes for result field") do
+      adapter_setup(adapter)
+      transformer = Factory(:transformer, :code => %{value}, :result_type => 'same')
+      field = @resource.fields_dataset[:name => 'first_name']
+      transformation = Transformation.create({
+        :transformer => transformer,
+        :resource => @resource,
+        :source_field => field,
+        :result_field_attributes => { :name => 'new_first_name' }
+      })
+
+      result_field = transformation.result_field.refresh
+      assert_equal field[:type], result_field[:type]
+      assert_equal field[:db_type], result_field[:db_type]
+      assert result_field[:is_generated]
+    end
+
+  end
+end

data/test/integration/test_transforming.rb

@@ -0,0 +1,154 @@
+require 'helper'
+
+class TestTransforming < Coupler::Test::IntegrationTest
+
+  def self.startup
+    super
+    Connection.delete
+    each_adapter do |adapter, config|
+      data = Array.new(50) { [Forgery(:name).first_name, Forgery(:name).last_name] }
+      conn = new_connection(adapter, :name => "#{adapter} connection").save!
+      conn.database do |db|
+        db.create_table!(:test_data) do
+          primary_key :id
+          String :first_name
+          String :last_name
+        end
+        db[:test_data].import([:first_name, :last_name], data)
+      end
+    end
+  end
+
+  def setup
+    super
+    @project = Project.create(:name => "Test project")
+  end
+
+  def adapter_setup(adapter)
+    @connection = new_connection(adapter, :name => "#{adapter} connection").save!
+    @resource = Resource.create(:name => "Test resource", :table_name => 'test_data', :project => @project, :connection => @connection)
+    @transformer = Transformer.create(:name => "downcaser", :allowed_types => %w{string}, :code => 'value.downcase', :result_type => 'same')
+  end
+
+  each_adapter do |adapter, _|
+    adapter_test(adapter, "single transformation") do
+      adapter_setup(adapter)
+      transformation = Transformation.create({
+        :resource => @resource, :transformer => @transformer,
+        :source_field => @resource.fields_dataset[:name => 'first_name']
+      })
+
+      @resource.transform!
+      assert @resource.transformed_at
+      assert_equal "#{transformation.id}", @resource.transformed_with
+      assert_equal "ok", @resource.status
+
+      @connection.database do |source_db|
+        @project.local_database do |local_db|
+          table_name = :"resource_#{@resource.id}"
+          assert_equal :string, local_db.schema(table_name).assoc(:first_name)[1][:type]
+          local_ds = local_db[table_name]
+          source_db[:test_data].each do |source_row|
+            local_row = local_ds[:id => source_row[:id]]
+            assert_equal source_row[:first_name].downcase, local_row[:first_name]
+          end
+        end
+      end
+    end
+
+    adapter_test(adapter, "transform goes in order of position") do
+      adapter_setup(adapter)
+      first_name = @resource.fields_dataset[:name => 'first_name']
+      foo_transformer = Transformer.create(:name => "foo", :allowed_types => %w{string}, :code => '"FOO"', :result_type => 'same')
+      xformation_1 = Transformation.create(:resource => @resource, :transformer => @transformer, :source_field => first_name)
+      xformation_2 = Transformation.create(:resource => @resource, :transformer => foo_transformer, :source_field => first_name)
+      xformation_1.update(:position => 2)
+      xformation_2.update(:position => 1)
+
+      @resource.transform!
+      @resource.local_dataset do |ds|
+        ds.each { |r| assert_equal "foo", r[:first_name] }
+      end
+    end
+
+    adapter_test(adapter, "transform into new result field") do
+      adapter_setup(adapter)
+      transformation = Transformation.create({
+        :resource => @resource, :transformer => @transformer,
+        :source_field => @resource.fields_dataset[:name => 'first_name'],
+        :result_field_attributes => { :name => 'downcased_first_name' }
+      })
+      @resource.transform!
+      @resource.local_dataset do |ds|
+        ds.each { |r| assert_equal r[:first_name].downcase, r[:downcased_first_name] }
+      end
+    end
+
+    adapter_test(adapter, "transforming only gets specified columns") do
+      adapter_setup(adapter)
+      @resource.fields_dataset.filter(["name IN ?", %w{last_name}]).update(:is_selected => false)
+      transformation = Transformation.create({
+        :resource => @resource, :transformer => @transformer,
+        :source_field => @resource.fields_dataset[:name => 'first_name']
+      })
+      @resource.transform!
+      @project.local_database do |db|
+        schema = db.schema(:"resource_#{@resource.id}")
+        assert_equal [:id, :first_name], schema.collect(&:first)
+      end
+    end
+
+    adapter_test(adapter, "preview inplace transformation") do
+      adapter_setup(adapter)
+      strlen = Transformer.create(:name => "strlen", :code => 'value.length', :allowed_types => %w{string}, :result_type => 'integer')
+      square = Transformer.create(:name => 'square', :code => "value ** 2", :allowed_types => %w{integer}, :result_type => 'integer')
+      first_name = @resource.fields_dataset[:name => 'first_name']
+      transformation_1 = Transformation.create({
+        :resource => @resource, :transformer => strlen,
+        :source_field => first_name
+      })
+      transformation_2 = Transformation.new({
+        :resource => @resource, :transformer => square,
+        :source_field => first_name, :result_field => first_name
+      })
+      arr = @resource.preview_transformation(transformation_2)
+      assert_equal [:id, :first_name, :last_name], arr[:fields]
+      assert_equal 50, arr[:data].length
+      @resource.source_dataset do |ds|
+        ds.limit(50).each_with_index do |row, i|
+          before = row.merge(:first_name => row[:first_name].length)
+          after  = before.merge(:first_name => before[:first_name] ** 2)
+          result = arr[:data].find { |r| r[:before][:id] == row[:id] }
+          assert_equal before, result[:before], "Before didn't match"
+          assert_equal after, result[:after], "After didn't match"
+        end
+      end
+    end
+
+    adapter_test(adapter, "preview transformation with invalid transformer") do
+      adapter_setup(adapter)
+      first_name = @resource.fields_dataset[:name => 'first_name']
+      transformation = Transformation.new({
+        :resource => @resource, :source_field => first_name,
+        :result_field => first_name, :transformer => nil,
+        :transformer_attributes => { :name => "foo", :code => "HAY", :allowed_types => %w{string}, :result_type => "string" }
+      })
+      result = @resource.preview_transformation(transformation)
+      assert_kind_of Exception, result
+    end
+
+    adapter_test(adapter, "transform with progress callback") do
+      adapter_setup(adapter)
+      first_name = @resource.fields_dataset[:name => 'first_name']
+      transformation = Transformation.create({
+        :resource => @resource, :source_field => first_name,
+        :result_field => first_name, :transformer => @transformer
+      })
+      count = 0
+      @resource.transform! do |n|
+        count += 1
+      end
+      assert count > 0
+    end
+  end
+end