chicago-etl 0.0.12 → 0.0.13

data/VERSION

@@ -1 +1 @@
-0.0.12
+0.0.13

data/chicago-etl.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "chicago-etl"
-  s.version = "0.0.12"
+  s.version = "0.0.13"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Roland Swingler"]
-  s.date = "2013-02-26"
+  s.date = "2013-04-16"
   s.description = "ETL tools for Chicago"
   s.email = "roland.swingler@gmail.com"
   s.extra_rdoc_files = [

data/lib/chicago/etl/key_builder.rb

@@ -29,6 +29,8 @@ module Chicago
             
             if table.identifiable?
               IdentifiableDimensionKeyBuilder.new(key_table, key_sink)
+            elsif existing_hash_column?(table)
+              ExistingHashColumnKeyBuilder.new(key_table, key_sink)
             else
               HashingKeyBuilder.new(key_table, key_sink, columns_to_hash)
             end
@@ -39,6 +41,10 @@ module Chicago
 
         private
 
+        def existing_hash_column?(table)
+          table.columns.any? {|c| c.binary? && c.name == :hash && c.unique? }
+        end
+
         def dimension?
           table.kind_of?(Chicago::Schema::Dimension)
         end
@@ -135,6 +141,24 @@ module Chicago
       end
     end
 
+    # Key builder for dimensions with a single unique hash column
+    # already present.
+    #
+    # @api private
+    class ExistingHashColumnKeyBuilder < KeyBuilder
+      def original_key(row)
+        row[:hash].upcase
+      end
+
+      def key_for_insert(original_id)
+        ("0x" + original_id).lit
+      end
+
+      def original_key_select_fragment
+        :hex.sql_function(:original_id).as(:original_id)
+      end
+    end
+
     # Key builder for dimensions with natuaral keys, but no simple
     # key.
     #

data/spec/etl/key_builder_spec.rb

@@ -22,6 +22,14 @@ describe Chicago::ETL::KeyBuilder do
       end
     end
 
+    @schema.define_dimension(:with_hash) do
+      columns do
+        binary :hash, :unique => true
+      end
+
+      natural_key :hash
+    end
+
     @schema.define_fact(:addresses) do
       dimensions :user, :address
       natural_key :user, :address
@@ -114,6 +122,35 @@ describe Chicago::ETL::KeyBuilder do
     end
   end
 
+  describe "for non-identifiable dimensions with an existing hash" do
+    before :each do
+      @builder = described_class.
+        for_table(@schema.dimension(:with_hash), @db)
+    end
+
+    it "returns an incrementing key, given a row" do
+      @builder.key(:hash => "aaa").should == 1
+      @builder.key(:hash => "aab").should == 2
+    end
+
+    it "returns the same incrementing key" do
+      @builder.key(:hash => "aaa").should == 1
+      @builder.key(:hash => "aaa").should == 1
+    end
+
+    it "returns the same incrementing key, ignoring case" do
+      @builder.key(:hash => "aaa").should == 1
+      @builder.key(:hash => "AAA").should == 1
+    end
+
+    it "inserts the hash as a binary literal" do
+      # Yuck. Don't like the implementation test, but mock
+      # expectations fail here for some reason, maybe because of the
+      # Sequel::LiteralString?
+      @builder.key_for_insert(@builder.original_key(:hash => "aaa")).should == "0xAAA".lit
+    end
+  end
+
   describe "for non-identifiable dimensions with natural keys" do
     before :each do
       @builder = described_class.for_table(@schema.dimension(:address), @db)

data/spec/etl/sequel/load_data_infile_expression_spec.rb

@@ -48,9 +48,13 @@ describe Chicago::ETL::SequelExtensions::LoadDataInfileExpression do
   end
 
   it "can set column values" do
-    described_class.new("bar.csv", :foo, ['@bar', 'quux'], 
+    sql = described_class.new("bar.csv", :foo, ['@bar', 'quux'], 
                         :set => {:bar => :unhex.sql_function("@bar".lit),
                         :etl_batch_id => 3}).
-      to_sql(TEST_DB).should include("SET `bar` = unhex(@bar), `etl_batch_id` = 3")
+      to_sql(TEST_DB)
+
+    sql.should include("SET")
+    sql.should include("`etl_batch_id` = 3")
+    sql.should include("`bar` = unhex(@bar)")
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: chicago-etl
 version: !ruby/object:Gem::Version 
-  hash: 7
+  hash: 5
   prerelease: 
   segments: 
   - 0
   - 0
-  - 12
-  version: 0.0.12
+  - 13
+  version: 0.0.13
 platform: ruby
 authors: 
 - Roland Swingler
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-02-26 00:00:00 Z
+date: 2013-04-16 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   version_requirements: &id001 !ruby/object:Gem::Requirement