dataduck 0.6.7 → 0.6.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5944e3a23d61e1bcdbfa5c82e1a8d0ee3fe8b4cf
-  data.tar.gz: 70c2e2b7af5377e13f473a9f7f56df438e5b6491
+  metadata.gz: e07cb92ca8f7f36c672dfb89e617e41ddb92f302
+  data.tar.gz: 40037cb63d58385f830043257883322e7dad1dbb
 SHA512:
-  metadata.gz: af935179e9a4a52d56423cb2a785db6d51421caaaba6d5fa33b6283708b6199252770892446b394c826fab07d75e57553c43d8ccf15754156fe063fc48732e54
-  data.tar.gz: 51ef2d29c8092aaf30e65326701ed3936c3e77a745ad171dbfd6d993c608e2312344258b31f0c525dd607739ae4376471ef3017858fbb50e6772bf4481b4b263
+  metadata.gz: bfce8a29678d44a96f26b05c3cef1fe047776976ced0d856a028276f9f65c0b8094f0ac64405f1eb8af91c1c2c07205b15776a448e9475d52c573596455409b2
+  data.tar.gz: 55a7dc9934972cb5953bac4e416dfddf26d4ae65389022168e8e8e36a6d6ffc74c4d76ce57eca9289bbc7a677061a093ec1f809a398b1838b776f0d0975589c6

data/dataduck.gemspec

@@ -29,4 +29,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "typhoeus", "~> 0.8"
   spec.add_runtime_dependency "oj", "~> 2.12"
   spec.add_runtime_dependency "sequel-redshift"
+  spec.add_runtime_dependency "whenever", "~> 0.9"
 end

data/lib/dataduck/commands.rb

@@ -89,28 +89,35 @@ module DataDuck
       which_database.dbconsole
     end
 
-    def self.etl(what = nil)
-      if what.nil?
+    def self.etl(*table_names_underscore)
+      if table_names_underscore.length == 0
         puts "You need to specify a table name or 'all'. Usage: dataduck etl all OR datduck etl my_table_name"
         return
       end
 
       only_destination = DataDuck::Destination.only_destination
 
-      if what == "all"
+      if table_names_underscore.length == 1 && table_names_underscore[0] == "all"
         etl = ETL.new(destinations: [only_destination], autoload_tables: true)
         etl.process!
       else
-        table_name_camelized = DataDuck::Util.underscore_to_camelcase(what)
-        require DataDuck.project_root + "/src/tables/#{ what }.rb"
-        table_class = Object.const_get(table_name_camelized)
-        if !(table_class <= DataDuck::Table)
-          raise Exception.new("Table class #{ table_name_camelized } must inherit from DataDuck::Table")
+        tables = []
+        table_names_underscore.each do |table_name|
+          table_name_camelized = DataDuck::Util.underscore_to_camelcase(table_name)
+          require DataDuck.project_root + "/src/tables/#{ table_name }.rb"
+          table_class = Object.const_get(table_name_camelized)
+          if !(table_class <= DataDuck::Table)
+            raise Exception.new("Table class #{ table_name_camelized } must inherit from DataDuck::Table")
+          end
+          table = table_class.new
+          tables << table
         end
-
-        table = table_class.new
-        etl = ETL.new(destinations: [only_destination], autoload_tables: false, tables: [table])
-        etl.process_table!(table)
+        etl = ETL.new({
+            destinations: [only_destination],
+            autoload_tables: false,
+            tables: tables
+        })
+        etl.process!
       end
     end
 
@@ -305,6 +312,13 @@ source1_password=#{ source_password }
       DataDuck::Commands.quickstart_save_file("#{ DataDuck.project_root }/src/tables/#{ table_name }.rb", result)
     end
 
+    def self.quickstart_create_schedule_config
+      namespace = Namespace.new
+      template = File.open("#{ DataDuck.gem_root }/lib/templates/quickstart/schedule.rb.erb", 'r').read
+      result = ERB.new(template).result(namespace.get_binding)
+      DataDuck::Commands.quickstart_save_file("#{ DataDuck.project_root }/config/schedule.rb", result)
+    end
+
     def self.quickstart_save_file(output_path_full, contents)
       *output_path, output_filename = output_path_full.split('/')
       output_path = output_path.join("/")

data/lib/dataduck/etl.rb

@@ -40,22 +40,27 @@ module DataDuck
       destinations_to_use = destinations_to_use.concat(self.class.destinations)
       destinations_to_use = destinations_to_use.concat(self.destinations)
       destinations_to_use.uniq!
-
-      @tables.each do |table_class|
-        table_to_etl = table_class.new
-        table_to_etl.etl!(destinations_to_use)
+      if destinations_to_use.length == 0
+        destinations_to_use << DataDuck::Destination.only_destination
       end
-    end
 
-    def process_table!(table)
-      Logs.info("Processing ETL for table #{ table.name } on pid #{ Process.pid }...")
+      errored_tables = []
 
-      destinations_to_use = []
-      destinations_to_use = destinations_to_use.concat(self.class.destinations)
-      destinations_to_use = destinations_to_use.concat(self.destinations)
-      destinations_to_use.uniq!
+      @tables.each do |table_or_class|
+        table = table_or_class.kind_of?(DataDuck::Table) ? table_or_class : table_or_class.new
+        Logs.info("Processing table '#{ table.name }'...")
+        begin
+          table.etl!(destinations_to_use)
+        rescue Exception => err
+          Logs.error("Error while processing table '#{ table.name }': #{ err.to_s }\n#{ err.backtrace.join("\n") }")
+          errored_tables << table
+        end
+      end
 
-      table.etl!(destinations_to_use)
+      Logs.info("Finished ETL processing for pid #{ Process.pid }, #{ @tables.length - errored_tables.length } succeeded, #{ errored_tables.length } failed")
+      if errored_tables.length > 0
+        Logs.info("The following tables encountered errors: '#{ errored_tables.map(&:name).join("', '") }'")
+      end
     end
   end
 end

data/lib/dataduck/version.rb

@@ -2,7 +2,7 @@ module DataDuck
   if !defined?(DataDuck::VERSION)
     VERSION_MAJOR = 0
     VERSION_MINOR = 6
-    VERSION_PATCH = 7
+    VERSION_PATCH = 8
     VERSION = [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH].join('.')
   end
 end

data/lib/integrations/semrush/organic_results.rb

@@ -45,8 +45,6 @@ module DataDuck
         escaped_phrase = URI.escape(phrase)
         semrush_api_url = "http://api.semrush.com/?type=phrase_organic&key=#{ self.key }&display_limit=#{ self.display_limit }&export_columns=Dn,Ur&phrase=#{ escaped_phrase }&database=#{ self.search_database }"
 
-        puts semrush_api_url
-
         response = Typhoeus.get(semrush_api_url)
         if response.response_code != 200
           raise OrganicResultsAPIError.new("SEMrush API for phrase #{ phrase } returned error #{ response.response_code } #{ response.body }")

data/lib/templates/quickstart/schedule.rb.erb

@@ -0,0 +1,19 @@
+# This file configures the whenver gem:
+# https://github.com/javan/whenever
+#
+# To update crontab from this schedule.rb file:
+#   whenever --update-crontab etl
+#
+# To clear the crontab from this schedule.rb file:
+#   whenever --clear-crontab etl
+#
+# To list out your crontab:
+#   crontab -l
+
+require 'dataduck'
+
+set :output, "#{ DataDuck.project_root }/log/cron.log"
+
+every 1.day, :at => '2:00 am' do
+  command "cd #{ DataDuck.project_root } && dataduck etl all"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dataduck
 version: !ruby/object:Gem::Version
-  version: 0.6.7
+  version: 0.6.8
 platform: ruby
 authors:
 - Jeff Pickhardt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-08 00:00:00.000000000 Z
+date: 2015-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -164,6 +164,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: whenever
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
 description: A straightforward, effective ETL framework.
 email:
 - pickhardt@gmail.com
@@ -226,6 +240,7 @@ files:
 - lib/integrations/optimizely/projects.rb
 - lib/integrations/optimizely/variations.rb
 - lib/integrations/semrush/organic_results.rb
+- lib/templates/quickstart/schedule.rb.erb
 - lib/templates/quickstart/table.rb.erb
 - static/logo.png
 homepage: http://dataducketl.com/