dataduck 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ef6f3cd5a8054cf855b227324845f2ec365516dd
-  data.tar.gz: 45030532745d7a68988bee5e95a791ed55bae6b0
+  metadata.gz: 497a92b4bfb99dba50ee81efb9bdf2b5b802023c
+  data.tar.gz: 7a5b3ef29d7bf2856b3ba196ed8a6cf25158b3f1
 SHA512:
-  metadata.gz: 1d167785a5f64fd8ea77546dcb3f7d19107c3c651aac5933afe09e7eee4eb1674c25a9223b1c724a8229334d669c41604288c29fa4da6e35bfa596ad8bde6a90
-  data.tar.gz: 708a7eb4404cee131bcd946314b57febaa6908c716c4ce2928167c88a0a61f084820f34258a3b5b74cbc97a215759e8cec13e9f89ff25a17aba2453a636d4517
+  metadata.gz: 89c16aa7ca78cbed8522f6c137eb4a3b0dd6ae6a87c3466b39ba937f319ce3d3d0db8370557d6259c6157ed2150ed94072a7489d6811bb3ad7908333fa0483f0
+  data.tar.gz: 314e2a64179b39b0f0f4d32fcdd5522a4b8b8574df9994c97956c5ff3066cde5c59e1a5a72d0c4c296594ee75ecaf4c30374c6d4a585a72d8b03ab80055fdfc2

data/lib/dataduck.rb

@@ -31,7 +31,7 @@ module DataDuck
   detect_project_root = Dir.getwd
   while true
     if detect_project_root == ""
-      raise Exception.new("Could not find a Gemfile in the current working directory or any parent directories. Are you sure you're running this from the right place?")
+      raise "Could not find a Gemfile in the current working directory or any parent directories. Are you sure you're running this from the right place?"
     end
 
     if File.exist?(detect_project_root + '/Gemfile')

data/lib/dataduck/commands.rb

@@ -2,6 +2,7 @@ require 'erb'
 require 'yaml'
 require 'fileutils'
 require 'typhoeus'
+require 'io/console'
 
 module DataDuck
   class Commands
@@ -49,7 +50,7 @@ module DataDuck
 
       begin
         DataDuck::Commands.public_send(command, *args[1..-1])
-      rescue Exception => err
+      rescue => err
         DataDuck::Logs.error(err)
       end
     end
@@ -111,7 +112,7 @@ module DataDuck
           require DataDuck.project_root + "/src/tables/#{ table_name }.rb"
           table_class = Object.const_get(table_name_camelized)
           if !(table_class <= DataDuck::Table)
-            raise Exception.new("Table class #{ table_name_camelized } must inherit from DataDuck::Table")
+            raise "Table class #{ table_name_camelized } must inherit from DataDuck::Table"
           end
           table = table_class.new
           tables << table
@@ -135,7 +136,7 @@ module DataDuck
       require DataDuck.project_root + "/src/tables/#{ table_name }.rb"
       table_class = Object.const_get(table_name_camelized)
       if !(table_class <= DataDuck::Table)
-        raise Exception.new("Table class #{ table_name_camelized } must inherit from DataDuck::Table")
+        raise "Table class #{ table_name_camelized } must inherit from DataDuck::Table"
       end
       table = table_class.new
       table.recreate!(DataDuck::Destination.only_destination)
@@ -157,7 +158,7 @@ module DataDuck
         require DataDuck.project_root + "/src/tables/#{ table_name }.rb"
         table_class = Object.const_get(table_name_camelized)
         if !(table_class <= DataDuck::Table)
-          raise Exception.new("Table class #{ table_name_camelized } must inherit from DataDuck::Table")
+          raise "Table class #{ table_name_camelized } must inherit from DataDuck::Table"
         end
 
         table = table_class.new

data/lib/dataduck/database.rb

@@ -7,15 +7,15 @@ module DataDuck
     end
 
     def connection
-      raise Exception.new("Must implement connection in subclass.")
+      raise "Must implement connection in subclass."
     end
 
     def query(sql)
-      raise Exception.new("Must implement query in subclass.")
+      raise "Must implement query in subclass."
     end
 
     def table_names
-      raise Exception.new("Must implement query in subclass.")
+      raise "Must implement query in subclass."
     end
 
     protected

data/lib/dataduck/destination.rb

@@ -20,18 +20,18 @@ module DataDuck
 
     def self.destination_config(name)
       if DataDuck.config['destinations'].nil? || DataDuck.config['destinations'][name.to_s].nil?
-        raise Exception.new("Could not find destination #{ name } in destinations configs.")
+        raise "Could not find destination #{ name } in destinations configs."
       end
 
       DataDuck.config['destinations'][name.to_s]
     end
 
     def load_table!(table)
-      raise Exception.new("Must implement load_table! in subclass")
+      raise "Must implement load_table! in subclass"
     end
 
     def recreate_table!(table)
-      raise Exception.new("Must implement load_table! in subclass")
+      raise "Must implement recreate_table! in subclass"
     end
 
     def postprocess!(table)
@@ -46,7 +46,7 @@ module DataDuck
       elsif allow_nil
         return nil
       else
-        raise Exception.new("Could not find destination #{ name } in destination configs.")
+        raise "Could not find destination #{ name } in destination configs."
       end
     end
 

data/lib/dataduck/etl.rb

@@ -51,7 +51,7 @@ module DataDuck
         Logs.info("Processing table '#{ table.name }'...")
         begin
           table.etl!(destinations_to_use)
-        rescue Exception => err
+        rescue => err
           Logs.error("Error while processing table '#{ table.name }': #{ err.to_s }\n#{ err.backtrace.join("\n") }")
           errored_tables << table
         end

data/lib/dataduck/redshift_destination.rb

@@ -203,7 +203,7 @@ module DataDuck
       Logs.debug("SQL executing on #{ self.name }:\n  " + sql)
       begin
         self.connection[sql].map { |elem| elem }
-      rescue Exception => err
+      rescue => err
         if err.to_s.include?("Check 'stl_load_errors' system table for details")
           self.raise_stl_load_error!
         else
@@ -266,7 +266,7 @@ module DataDuck
       DataDuck::Logs.info "Recreating table #{ table.name }..."
 
       if !self.table_names.include?(table.name)
-        raise Exception.new("Table #{ table.name } doesn't exist on the Redshift database, so it can't be recreated. Did you want to use `dataduck create #{ table.name }` instead?")
+        raise "Table #{ table.name } doesn't exist on the Redshift database, so it can't be recreated. Did you want to use `dataduck create #{ table.name }` instead?"
       end
 
       recreating_temp_name = "zz_dataduck_recreating_#{ table.name }"

data/lib/dataduck/s3_object.rb

@@ -50,7 +50,7 @@ module DataDuck
             })
         begin
           response = s3.put_object(put_hash)
-        rescue Exception => e
+        rescue => e
           if attempts == S3Object.max_retries
             throw e
           end

data/lib/dataduck/source.rb

@@ -22,7 +22,7 @@ module DataDuck
 
     def self.source_config(name)
       if DataDuck.config['sources'].nil? || DataDuck.config['sources'][name.to_s].nil?
-        raise Exception.new("Could not find source #{ name } in source configs.")
+        raise "Could not find source #{ name } in source configs."
       end
 
       DataDuck.config['sources'][name.to_s]
@@ -36,7 +36,7 @@ module DataDuck
       elsif allow_nil
         return nil
       else
-        raise Exception.new("Could not find source #{ name } in source configs.")
+        raise "Could not find source #{ name } in source configs."
       end
     end
 

data/lib/dataduck/table.rb

@@ -57,8 +57,8 @@ module DataDuck
 
     def check_table_valid!
       if !self.batch_size.nil?
-        raise Exception.new("Table #{ self.name }'s batch_size must be > 0") unless self.batch_size > 0
-        raise Exception.new("Table #{ self.name } has batch_size defined but no extract_by_column") if self.extract_by_column.nil?
+        raise "Table #{ self.name }'s batch_size must be > 0" unless self.batch_size > 0
+        raise "Table #{ self.name } has batch_size defined but no extract_by_column" if self.extract_by_column.nil?
       end
     end
 
@@ -87,12 +87,17 @@ module DataDuck
         destination.drop_staging_table!(self)
       end
 
+      data_processed = false
       batch_number = 0
       while batch_number < 1_000
         batch_number += 1
         self.extract!(destination, options)
-        self.transform!
-        self.load!(destination)
+        if self.data.length > 0
+          self.transform!
+          self.data.compact!
+          self.load!(destination) if self.data.length > 0
+          data_processed = true
+        end
 
         if self.batch_size.nil?
           break
@@ -108,11 +113,15 @@ module DataDuck
 
       self.data = []
 
-      if self.should_fully_reload?
-        destination.finish_fully_reloading_table!(self)
-      end
+      if data_processed
+        if self.should_fully_reload?
+          destination.finish_fully_reloading_table!(self)
+        end
 
-      self.postprocess!(destination, options)
+        self.postprocess!(destination, options)
+      else
+        DataDuck::Logs.info "No data extracted for table #{ self.name }"
+      end
     end
 
     def extract!(destination = nil, options = {})

data/lib/dataduck/version.rb

@@ -1,7 +1,7 @@
 module DataDuck
   if !defined?(DataDuck::VERSION)
     VERSION_MAJOR = 0
-    VERSION_MINOR = 7
+    VERSION_MINOR = 8
     VERSION_PATCH = 0
     VERSION = [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH].join('.')
   end

data/lib/integrations/optimizely/optimizely_integration.rb

@@ -43,7 +43,7 @@ module DataDuck
           experiment_variations = []
           begin
             experiment_variations = fetch_data(endpoint)
-          rescue Exception => err
+          rescue => err
             broken_experiments << experiment
           end
           experiment_variations.each do |exp_var|
@@ -69,7 +69,7 @@ module DataDuck
 
         response = Typhoeus.get("https://www.optimizelyapis.com/experiment/v1/#{ api_endpoint }", headers: {'Token' => optimizely_api_token})
         if response.response_code != 200
-          raise Exception.new("Optimizely API for #{ api_endpoint } returned error #{ response.response_code} #{ response.body }")
+          raise "Optimizely API for #{ api_endpoint } returned error #{ response.response_code} #{ response.body }"
         end
 
         rows = Oj.load(response.body)

data/lib/integrations/semrush/organic_results.rb

@@ -4,7 +4,7 @@ require 'uri'
 
 module DataDuck
   module SEMRush
-    class OrganicResultsAPIError < Exception; end
+    class OrganicResultsAPIError < StandardError; end
 
     class OrganicResults < DataDuck::IntegrationTable
       def display_limit
@@ -16,7 +16,7 @@ module DataDuck
       end
 
       def phrases
-        raise Exception("Must implement phrases method to be an array of the phrases you want.")
+        raise "Must implement phrases method to be an array of the phrases you want."
       end
 
       def prefix

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dataduck
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Jeff Pickhardt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-06 00:00:00.000000000 Z
+date: 2016-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler