dataduck 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 497a92b4bfb99dba50ee81efb9bdf2b5b802023c
-  data.tar.gz: 7a5b3ef29d7bf2856b3ba196ed8a6cf25158b3f1
+  metadata.gz: be1cff2044ac902e2482b8cf432fdfded2b69ca6
+  data.tar.gz: dcf5456ce914d227b25e002a7bf4b10b8f0892e7
 SHA512:
-  metadata.gz: 89c16aa7ca78cbed8522f6c137eb4a3b0dd6ae6a87c3466b39ba937f319ce3d3d0db8370557d6259c6157ed2150ed94072a7489d6811bb3ad7908333fa0483f0
-  data.tar.gz: 314e2a64179b39b0f0f4d32fcdd5522a4b8b8574df9994c97956c5ff3066cde5c59e1a5a72d0c4c296594ee75ecaf4c30374c6d4a585a72d8b03ab80055fdfc2
+  metadata.gz: 49ebaa72e9ca8eb7dc503853d5b8829c03c5da17cbc4bd391a4c0076a922046cded8002acfc13ed70b6d1f572ead6102d5b3e43e94d7c6981ac8864534dfe07d
+  data.tar.gz: c5b3e2c56c5d323a8876792f56be753a743860edc8978c06fb3126fd17949e01797b16e13a3d735b90e478251ddc455f820395b0093a8e4be4bd403c2581e5df

data/lib/dataduck/redshift_destination.rb

@@ -55,7 +55,7 @@ module DataDuck
     def create_columns_on_data_warehouse!(table)
       columns = get_columns_in_data_warehouse(table.building_name)
       column_names = columns.map { |col| col[:name].to_s }
-      table.output_schema.map do |name, data_type|
+      table.create_schema.map do |name, data_type|
         if !column_names.include?(name.to_s)
           redshift_data_type = self.type_to_redshift_type(data_type)
           self.query("ALTER TABLE #{ table.building_name } ADD #{ name } #{ redshift_data_type }")
@@ -65,7 +65,7 @@ module DataDuck
 
     def create_table_query(table, table_name = nil)
       table_name ||= table.name
-      props_array = table.output_schema.map do |name, data_type|
+      props_array = table.create_schema.map do |name, data_type|
         redshift_data_type = self.type_to_redshift_type(data_type)
         "\"#{ name }\" #{ redshift_data_type }"
       end
@@ -253,7 +253,8 @@ module DataDuck
       DataDuck::Logs.info "Loading table #{ table.name }..."
       s3_object = self.upload_table_to_s3!(table)
       self.create_output_tables!(table)
-      self.query(self.copy_query(table, s3_object.s3_path))
+      query_to_run = self.copy_query(table, s3_object.s3_path)
+      self.query(query_to_run)
       s3_object.delete!
 
       if table.staging_name != table.building_name
@@ -271,7 +272,7 @@ module DataDuck
 
       recreating_temp_name = "zz_dataduck_recreating_#{ table.name }"
       self.create_output_table_with_name!(table, recreating_temp_name)
-      self.query("INSERT INTO #{ recreating_temp_name } (\"#{ table.output_column_names.join('","') }\") SELECT \"#{ table.output_column_names.join('","') }\" FROM #{ table.name }")
+      self.query("INSERT INTO #{ recreating_temp_name } (\"#{ table.create_column_names.join('","') }\") SELECT \"#{ table.create_column_names.join('","') }\" FROM #{ table.name }")
       self.query("ALTER TABLE #{ table.name } RENAME TO zz_dataduck_recreating_old_#{ table.name }")
       self.query("ALTER TABLE #{ recreating_temp_name } RENAME TO #{ table.name }")
       self.query("DROP TABLE zz_dataduck_recreating_old_#{ table.name }")

data/lib/dataduck/table.rb

@@ -212,7 +212,11 @@ module DataDuck
     end
 
     def should_fully_reload?
-      false # Set to true if you want to fully reload a table with each ETL
+      false
+    end
+
+    def autogenerate_identity?
+      false
     end
 
     def building_name
@@ -223,12 +227,24 @@ module DataDuck
       "zz_dataduck_#{ self.name }"
     end
 
+    def create_schema
+      if self.autogenerate_identity?
+        Util.deep_merge(output_schema, {dataduck_identity: 'bigint identity(1, 1)'}) # Redshift only
+      else
+        output_schema
+      end
+    end
+
+    def create_column_names
+      self.create_schema.keys.map(&:to_s).sort
+    end
+
     def output_schema
       self.class.output_schema || self.class.superclass.output_schema || {}
     end
 
     def output_column_names
-      self.output_schema.keys.sort.map(&:to_s)
+      self.output_schema.keys.map(&:to_s).sort
     end
 
     def postprocess!(destination, options = {})

data/lib/dataduck/version.rb

@@ -1,7 +1,7 @@
 module DataDuck
   if !defined?(DataDuck::VERSION)
     VERSION_MAJOR = 0
-    VERSION_MINOR = 8
+    VERSION_MINOR = 9
     VERSION_PATCH = 0
     VERSION = [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH].join('.')
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dataduck
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Jeff Pickhardt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-25 00:00:00.000000000 Z
+date: 2016-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler