dataduck 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a8ecc21c43cf23e347238bd4eb130d4b0d2bddb
-  data.tar.gz: 3a66f7465e14f20a13c3ce07b1a023d9f5436e4f
+  metadata.gz: fb5bcf70fd0c35ad944220251f360767852eeb80
+  data.tar.gz: e73dd71f9a0761c56dee3637754e26ae962d210b
 SHA512:
-  metadata.gz: cefcb97553320a432f7f48dd586176519f4a381d215ce45516b92849e922fe8d87bc938ba65102b84e90c1b86f71b0bf35fb54106070cefb9b30c94a9e61f087
-  data.tar.gz: 1f0fd5859639ec367ae9dbabb622bbd82583282166473eeab47c97dc8a853bc32de7f4c2d5dbcff0fc31eed3dc06f4f978143e5ce581ad3fc4f69d0e63c51c54
+  metadata.gz: a607da3c47de0279fa521321555bc4b1218f5375a6ec0aaa2244472e0b42322dca01708a57c813be64d9b34f78ef4588982c38e76c8e7c8ff6c83efb774e0d93
+  data.tar.gz: f7742ec8eff3e4c8bc36fa5015b42ede4fb852be759cb9be5bae25091a13c74160ae93eb91ba2391e1ba0099a53340f17e0839050ece403b90021d363eec3334

data/dataduck.gemspec

@@ -26,5 +26,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "pg", '~> 0.16'
   spec.add_runtime_dependency "mysql2", '~> 0.4'
   spec.add_runtime_dependency "aws-sdk", "~> 2.0"
+  spec.add_runtime_dependency "typhoeus", "~> 0.8"
   spec.add_runtime_dependency "sequel-redshift"
 end

data/docs/commands/console.md

@@ -3,3 +3,7 @@
 The `console` command will place you into a Ruby console with DataDuck loaded. This can be useful for debugging. Run it with:
 
 `$ dataduck console`
+
+It is also aliased to simply "c":
+
+`$ dataduck c`

data/docs/commands/dbconsole.md

@@ -7,6 +7,10 @@ This will connect you with the destination (e.g. Redshift):
 
 `$ dataduck dbconsole`
 
+It is also aliased to simply "d":
+
+`$ dataduck d`
+
 You can also use one of these:
 
 `$ dataduck dbconsole source`

data/docs/commands/recreate.md

@@ -0,0 +1,21 @@
+# The `recreate` command
+
+The `recreate` command will create a brand new version of table on your destination, then move the existing data over to the new table.
+
+This is useful if you change the indexes or distribution keys of a table.
+
+It takes the following steps, so that the original table is affected for as little amount of time as needed:
+
+1. Creates a new table named zz_dataduck_recreating_(tablename)
+
+2. Moves the table data from the original table to the new table.
+
+3. Renames the original table to zz_dataduck_recreating_old_(tablename)
+
+4. Renames the zz_dataduck_recreating_(tablename) to tablename.
+
+5. Drops zz_dataduck_recreating_old_(tablename)
+
+To recreate a table, use the command:
+
+`$ dataduck recreate my_table_name`

data/docs/contents.yml

@@ -7,6 +7,7 @@
   "dbconsole": dbconsole
   "etl": etl
   "quickstart": quickstart
+  "recreate": recreate
   "show": show
 
 "Tables":

data/lib/dataduck/commands.rb

@@ -1,6 +1,7 @@
 require 'erb'
 require 'yaml'
 require 'fileutils'
+require 'typhoeus'
 
 module DataDuck
   class Commands
@@ -32,7 +33,7 @@ module DataDuck
     end
 
     def self.acceptable_commands
-      ['console', 'dbconsole', 'etl', 'quickstart', 'show']
+      ['c', 'console', 'd', 'dbconsole', 'etl', 'quickstart', 'recreate', 'show']
     end
 
     def self.route_command(args)
@@ -49,12 +50,20 @@ module DataDuck
       DataDuck::Commands.public_send(command, *args[1..-1])
     end
 
+    def self.c
+      self.console
+    end
+
     def self.console
       require "irb"
       ARGV.clear
       IRB.start
     end
 
+    def self.d(where = "destination")
+      self.dbconsole(where)
+    end
+
     def self.dbconsole(where = "destination")
       which_database = nil
       if where == "destination"
@@ -110,6 +119,17 @@ module DataDuck
       puts "Commands: #{ acceptable_commands.sort.join(' ') }"
     end
 
+    def self.recreate(table_name)
+      table_name_camelized = DataDuck::Util.underscore_to_camelcase(table_name)
+      require DataDuck.project_root + "/src/tables/#{ table_name }.rb"
+      table_class = Object.const_get(table_name_camelized)
+      if !(table_class <= DataDuck::Table)
+        raise Exception.new("Table class #{ table_name_camelized } must inherit from DataDuck::Table")
+      end
+      table = table_class.new
+      table.recreate!(DataDuck::Destination.only_destination)
+    end
+
     def self.show(table_name = nil)
       if table_name.nil?
         Dir[DataDuck.project_root + "/src/tables/*.rb"].each do |file|
@@ -134,10 +154,34 @@ module DataDuck
       end
     end
 
+    def self.quickstart_register_email(email)
+      registration_data = {
+          email: email,
+          version: DataDuck::VERSION,
+          source: "quickstart"
+      }
+
+      request = Typhoeus::Request.new(
+          "dataducketl.com/api/v1/register",
+          method: :post,
+          body: registration_data,
+          timeout: 30,
+          connecttimeout: 10,
+      )
+
+      hydra = Typhoeus::Hydra.new
+      hydra.queue(request)
+      hydra.run
+    end
+
     def self.quickstart
       puts "Welcome to DataDuck!"
       puts "This quickstart wizard will help you set up DataDuck."
 
+      puts "What is your work email address?"
+      email = STDIN.gets.strip
+      self.quickstart_register_email(email)
+
       puts "What kind of database would you like to source from?"
       db_type = prompt_choices([
           [:mysql, "MySQL"],
@@ -188,6 +232,11 @@ module DataDuck
       end
 
       config_obj = {
+        'users' => {
+            email => {
+                'admin' => true
+            }
+        },
         'sources' => {
           'source1' => {
             'type' => db_type.to_s,

data/lib/dataduck/destination.rb

@@ -30,6 +30,10 @@ module DataDuck
       raise Exception.new("Must implement load_table! in subclass")
     end
 
+    def recreate_table!(table)
+      raise Exception.new("Must implement load_table! in subclass")
+    end
+
     def self.destination(name, allow_nil = false)
       name = name.to_s
 

data/lib/dataduck/redshift_destination.rb

@@ -76,15 +76,19 @@ module DataDuck
     end
 
     def create_output_tables!(table)
-      self.query(self.create_table_query(table, table.building_name))
+      self.create_output_table_with_name!(table, table.building_name)
       self.create_columns_on_data_warehouse!(table)
 
       if table.building_name != table.staging_name
         self.drop_staging_table!(table)
-        self.query(self.create_table_query(table, table.staging_name))
+        self.create_output_table_with_name!(table, table.staging_name)
       end
     end
 
+    def create_output_table_with_name!(table, name)
+      self.query(self.create_table_query(table, name))
+    end
+
     def data_as_csv_string(data, property_names)
       data_string_components = [] # for performance reasons, join strings this way
       data.each do |result|
@@ -216,6 +220,21 @@ module DataDuck
       end
     end
 
+    def recreate_table!(table)
+      DataDuck::Logs.info "Recreating table #{ table.name }..."
+
+      if !self.table_names.include?(table.name)
+        raise Exception.new("Table #{ table.name } doesn't exist on the Redshift database, so it can't be recreated. Did you want to use `dataduck create #{ table.name }` instead?")
+      end
+
+      recreating_temp_name = "zz_dataduck_recreating_#{ table.name }"
+      self.create_output_table_with_name!(table, recreating_temp_name)
+      self.query("INSERT INTO #{ recreating_temp_name } (\"#{ table.output_column_names.join('","') }\") SELECT \"#{ table.output_column_names.join('","') }\" FROM #{ table.name }")
+      self.query("ALTER TABLE #{ table.name } RENAME TO zz_dataduck_recreating_old_#{ table.name }")
+      self.query("ALTER TABLE #{ recreating_temp_name } RENAME TO #{ table.name }")
+      self.query("DROP TABLE zz_dataduck_recreating_old_#{ table.name }")
+    end
+
     def self.value_to_string(value)
       string_value = ''
       if value.respond_to? :to_s

data/lib/dataduck/table.rb

@@ -79,7 +79,7 @@ module DataDuck
         batch_number += 1
         self.extract!(destination)
         self.transform!
-        destination.load_table!(self)
+        self.load!(destination)
 
         if self.batch_size.nil?
           break
@@ -152,6 +152,10 @@ module DataDuck
       end
     end
 
+    def load!(destination)
+      destination.load_table!(self)
+    end
+
     def indexes
       which_columns = []
       which_columns << "id" if self.output_column_names.include?("id")
@@ -189,6 +193,10 @@ module DataDuck
       self.output_schema.keys.sort.map(&:to_s)
     end
 
+    def recreate!(destination)
+      destination.recreate_table!(self)
+    end
+
     def show
       puts "Table #{ self.name }"
       self.class.sources.each do |source_spec|

data/lib/dataduck/version.rb

@@ -1,6 +1,8 @@
 module DataDuck
-  VERSION_MAJOR = 0
-  VERSION_MINOR = 6
-  VERSION_PATCH = 1
-  VERSION = [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH].join('.')
+  if !defined?(DataDuck::VERSION)
+    VERSION_MAJOR = 0
+    VERSION_MINOR = 6
+    VERSION_PATCH = 2
+    VERSION = [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH].join('.')
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dataduck
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Jeff Pickhardt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-28 00:00:00.000000000 Z
+date: 2015-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -122,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: sequel-redshift
   requirement: !ruby/object:Gem::Requirement
@@ -161,6 +175,7 @@ files:
 - docs/commands/dbconsole.md
 - docs/commands/etl.md
 - docs/commands/quickstart.md
+- docs/commands/recreate.md
 - docs/commands/show.md
 - docs/contents.yml
 - docs/overview/README.md