dataduck 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: de529cfe949f8c1fb4a4cb36129188636ffbcb74
-  data.tar.gz: ebbcaa35d0babcbabdaef339f9ef72b061fc54d5
+  metadata.gz: 19296dd9ee329eee67c43509d2ab0012a9f4f47f
+  data.tar.gz: b4518be78d421506a7e1820056f4b5022b79eec7
 SHA512:
-  metadata.gz: 2958e2909631c314c7104fa340f0a587b47ab172417aa792f2b1a31377b1c455188acb0554c84ab656d057c4c060f9d7110bb6971fe45cedc8d4d3a117339d1e
-  data.tar.gz: 5e62d009d64ebe30b1ade7184c5e7f1041e8c58d3cbf5be39d3d1885e89a3126870197568694ead1673018985b079757ad00117d1aa5423d6413d027a292cd09
+  metadata.gz: c00321b4699cb60d9786c7808a0dbb10acfa4cab820df0394397289087308fb3e96c185b388334b3d6caa0b41920e146f07454fdbbf7b04b8b5f534579ef434f
+  data.tar.gz: db88b4fa5354387fdae08f1b49382ae1aa3cfa97eefa735fb2e619e5ea8d46c47b77d61a311dd4f9c801bdb773bc007edc1d30d19d0ecb98fef073a7674113fa

data/DEV_README.md

@@ -14,4 +14,4 @@ gem push dataduck-VERSION.gem
 
 Use something like this:
 
-gem 'dataduck', '0.3.0', path: '/Users/jrp/projects/dataduck'
+gem 'dataduck', '0.5.1', path: '/Users/jrp/projects/dataduck'

data/lib/dataduck/destination.rb

@@ -1,3 +1,5 @@
+require_relative 'database'
+
 module DataDuck
   class Destination < DataDuck::Database
     def self.load_config!

data/lib/dataduck/logs.rb

@@ -12,14 +12,26 @@ module DataDuck
       @@logger ||= Logger.new(log_file_path, shift_age = 100, shift_size = 100 * @@ONE_MB_IN_BYTES)
     end
 
+    def Logs.debug(message)
+      self.ensure_logger_exists!
+      message = Logs.sanitize_message(message)
+
+      puts "[DEBUG] #{ message }"
+      @@logger.debug(message)
+    end
+
     def Logs.info(message)
       self.ensure_logger_exists!
+      message = Logs.sanitize_message(message)
+
       puts "[INFO] #{ message }"
       @@logger.info(message)
     end
 
     def Logs.warn(message)
       self.ensure_logger_exists!
+      message = Logs.sanitize_message(message)
+
       puts "[WARN] #{ message }"
       @@logger.warn(message)
     end
@@ -27,8 +39,20 @@ module DataDuck
     def Logs.error(err, message = nil)
       self.ensure_logger_exists!
       message = err.to_s unless message
+      message = Logs.sanitize_message(message)
+
       puts "[ERROR] #{ message }"
       @@logger.error(message)
     end
+
+    private
+
+      def Logs.sanitize_message(message)
+        message = message.gsub(/aws_access_key_id=[^';]+/, "aws_access_key_id=******")
+        message = message.gsub(/AWS_ACCESS_KEY_ID=[^';]+/, "AWS_ACCESS_KEY_ID=******")
+        message = message.gsub(/aws_secret_access_key=[^';]+/, "aws_secret_access_key=******")
+        message = message.gsub(/AWS_SECRET_ACCESS_KEY=[^';]+/, "AWS_SECRET_ACCESS_KEY=******")
+        message
+      end
   end
 end

data/lib/dataduck/mysql_source.rb

@@ -18,5 +18,9 @@ module DataDuck
 
       self.find_command_and_execute("mysql", *args)
     end
+
+    def escape_char
+      '`'
+    end
   end
 end

data/lib/dataduck/postgresql_source.rb

@@ -25,5 +25,9 @@ module DataDuck
       size_in_gb = size_in_bytes / 1_000_000_000.0
       size_in_gb
     end
+
+    def escape_char
+      '"'
+    end
   end
 end

data/lib/dataduck/redshift_destination.rb

@@ -1,4 +1,4 @@
-require_relative 'destination.rb'
+require_relative 'destination'
 
 module DataDuck
   class RedshiftDestination < DataDuck::Destination
@@ -43,8 +43,7 @@ module DataDuck
       column_names = columns.map { |col| col[:name].to_s }
       table.output_schema.map do |name, data_type|
         if !column_names.include?(name.to_s)
-          redshift_data_type = data_type.to_s
-          redshift_data_type = 'varchar(255)' if redshift_data_type == 'string'
+          redshift_data_type = self.type_to_redshift_type(data_type)
           self.query("ALTER TABLE #{ table.building_name } ADD #{ name } #{ redshift_data_type }")
         end
       end
@@ -53,8 +52,7 @@ module DataDuck
     def create_table_query(table, table_name = nil)
       table_name ||= table.name
       props_array = table.output_schema.map do |name, data_type|
-        redshift_data_type = data_type.to_s
-        redshift_data_type = 'varchar(255)' if redshift_data_type == 'string'
+        redshift_data_type = self.type_to_redshift_type(data_type)
         "\"#{ name }\" #{ redshift_data_type }"
       end
       props_string = props_array.join(', ')
@@ -99,6 +97,20 @@ module DataDuck
       return data_string_components.join
     end
 
+    def type_to_redshift_type(which_type)
+      which_type = which_type.to_s
+
+      if ["string", "text", "bigtext"].include?(which_type)
+        {
+            "string" => "varchar(255)",
+            "text" => "varchar(8191)",
+            "bigtext" => "varchar(65535)", # Redshift maximum
+        }[which_type]
+      else
+        which_type
+      end
+    end
+
     def dbconsole(options = {})
       args = []
       args << "--host=#{ @host }"
@@ -147,6 +159,7 @@ module DataDuck
     end
 
     def query(sql)
+      Logs.debug("SQL executing on #{ self.name }:\n  " + sql)
       self.connection[sql].map { |elem| elem }
     end
 

data/lib/dataduck/source.rb

@@ -1,3 +1,5 @@
+require_relative 'database'
+
 module DataDuck
   class Source < DataDuck::Database
     def self.load_config!
@@ -47,6 +49,10 @@ module DataDuck
       return DataDuck::Source.source(source_name)
     end
 
+    def escape_char
+      '' # implement in subclass, e.g. " in postgresql and ` in mysql
+    end
+
     def schema(table_name)
       self.connection.schema(table_name)
     end

data/lib/dataduck/sql_db_source.rb

@@ -1,4 +1,5 @@
-require_relative 'source.rb'
+require_relative 'source'
+require_relative 'logs'
 
 require 'sequel'
 
@@ -29,7 +30,11 @@ module DataDuck
     def db_type
       return @initialized_db_type if @initialized_db_type
 
-      raise Exception.new("Abstract method db_type must be overwritten by subclass, or passed as data when initializing.")
+      raise NotImplementedError.new("Abstract method db_type must be overwritten by subclass, or passed as data when initializing.")
+    end
+
+    def escape_char
+      raise NotImplementedError.new("Abstract method escape_char must be overwritten by subclass.")
     end
 
     def table_names
@@ -41,6 +46,7 @@ module DataDuck
         raise ArgumentError.new("Database #{ self.name } must not run mutating sql: #{ sql }")
       end
 
+      Logs.debug("SQL executing on #{ self.name }:\n  " + sql)
       self.connection.fetch(sql).all
     end
   end

data/lib/dataduck/table.rb

@@ -1,3 +1,5 @@
+require_relative 'logs'
+
 module DataDuck
   class Table
     class << self
@@ -113,8 +115,10 @@ module DataDuck
     end
 
     def extract_query(source_spec, destination = nil)
+      escape_char = source_spec[:source].escape_char
+
       base_query = source_spec.has_key?(:query) ? source_spec[:query] :
-         "SELECT \"#{ source_spec[:columns].sort.join('","') }\" FROM #{ source_spec[:table_name] }"
+         "SELECT #{ escape_char }#{ source_spec[:columns].sort.join(escape_char + ',' + escape_char) }#{ escape_char } FROM #{ source_spec[:table_name] }"
 
       extract_by_clause = ""
       limit_clause = ""
@@ -165,11 +169,11 @@ module DataDuck
     end
 
     def output_schema
-      self.class.output_schema
+      self.class.output_schema || {}
     end
 
     def output_column_names
-      self.class.output_schema.keys.sort.map(&:to_s)
+      self.output_schema.keys.sort.map(&:to_s)
     end
 
     def show

data/lib/dataduck/version.rb

@@ -1,6 +1,6 @@
 module DataDuck
   VERSION_MAJOR = 0
   VERSION_MINOR = 5
-  VERSION_PATCH = 0
+  VERSION_PATCH = 1
   VERSION = [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH].join('.')
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dataduck
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Jeff Pickhardt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-19 00:00:00.000000000 Z
+date: 2015-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler