dataduck 0.6.2 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fb5bcf70fd0c35ad944220251f360767852eeb80
-  data.tar.gz: e73dd71f9a0761c56dee3637754e26ae962d210b
+  metadata.gz: d72a9c14a46ac5a3377bd79b8952b61919d6a4e7
+  data.tar.gz: b527269b93a5b57057553330859ccda872150e36
 SHA512:
-  metadata.gz: a607da3c47de0279fa521321555bc4b1218f5375a6ec0aaa2244472e0b42322dca01708a57c813be64d9b34f78ef4588982c38e76c8e7c8ff6c83efb774e0d93
-  data.tar.gz: f7742ec8eff3e4c8bc36fa5015b42ede4fb852be759cb9be5bae25091a13c74160ae93eb91ba2391e1ba0099a53340f17e0839050ece403b90021d363eec3334
+  metadata.gz: c8bade98533f439afd0465f22e12ae98b56bb7a29343a76e714ddc79908b4feb19df928f3fb0ff470b5ce608800c2b43e7380f1a63ce52f8acd8b4697f182484
+  data.tar.gz: cde4b3cc18a2140330fc3ba8577efc0c4caecff7de5d2b36223b637421a0e7f1cdd7b25d9f47d6b53292efe015376a1fa6918659ddae2cccc8d646b9e9648421

data/dataduck.gemspec

@@ -27,5 +27,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "mysql2", '~> 0.4'
   spec.add_runtime_dependency "aws-sdk", "~> 2.0"
   spec.add_runtime_dependency "typhoeus", "~> 0.8"
+  spec.add_runtime_dependency "oj", "~> 2.12"
   spec.add_runtime_dependency "sequel-redshift"
 end

data/lib/dataduck.rb

@@ -11,6 +11,14 @@ Dir[File.dirname(__FILE__) + '/dataduck/*.rb'].each do |file|
   require file
 end
 
+Dir[File.dirname(__FILE__) + '/integrations/*.rb'].each do |file|
+  require file
+end
+
+Dir[File.dirname(__FILE__) + '/integrations/*/*.rb'].each do |file|
+  require file
+end
+
 module DataDuck
   extend ModuleVars
 

data/lib/dataduck/redshift_destination.rb

@@ -1,6 +1,8 @@
 require_relative 'destination'
 
 module DataDuck
+  class RedshiftLoadError < StandardError; end
+
   class RedshiftDestination < DataDuck::Destination
     attr_accessor :aws_key
     attr_accessor :aws_secret
@@ -45,7 +47,7 @@ module DataDuck
       query_fragments << "FROM '#{ s3_path }'"
       query_fragments << "CREDENTIALS 'aws_access_key_id=#{ self.aws_key };aws_secret_access_key=#{ self.aws_secret }'"
       query_fragments << "REGION '#{ self.s3_region }'"
-      query_fragments << "CSV TRUNCATECOLUMNS ACCEPTINVCHARS EMPTYASNULL"
+      query_fragments << "CSV IGNOREHEADER 1 TRUNCATECOLUMNS ACCEPTINVCHARS EMPTYASNULL"
       query_fragments << "DATEFORMAT 'auto'"
       return query_fragments.join(" ")
     end
@@ -90,10 +92,17 @@ module DataDuck
     end
 
     def data_as_csv_string(data, property_names)
-      data_string_components = [] # for performance reasons, join strings this way
+      data_string_components = [] # join strings this way for now, could be optimized later
+
+      data_string_components << property_names.join(',') # header column
+      data_string_components << "\n"
+
       data.each do |result|
         property_names.each_with_index do |property_name, index|
           value = result[property_name.to_sym]
+          if value.nil?
+            value = result[property_name.to_s]
+          end
 
           if index == 0
             data_string_components << '"'
@@ -176,7 +185,24 @@ module DataDuck
 
     def query(sql)
       Logs.debug("SQL executing on #{ self.name }:\n  " + sql)
-      self.connection[sql].map { |elem| elem }
+      begin
+        self.connection[sql].map { |elem| elem }
+      rescue Exception => err
+        if err.to_s.include?("Check 'stl_load_errors' system table for details")
+          self.raise_stl_load_error!
+        else
+          raise err
+        end
+      end
+    end
+
+    def raise_stl_load_error!
+      load_error_sql = "SELECT filename, line_number, colname, position, err_code, err_reason FROM stl_load_errors ORDER BY starttime DESC LIMIT 1"
+      load_error_details = self.connection[load_error_sql].map { |elem| elem }.first
+
+      raise RedshiftLoadError.new("Error loading Redshift, '#{ load_error_details[:err_reason].strip }' " +
+          "(code #{ load_error_details[:err_code] }) with file #{ load_error_details[:filename].strip } " +
+          "for column '#{ load_error_details[:colname].strip }'. The error occurred at line #{ load_error_details[:line_number] }, position #{ load_error_details[:position] }.")
     end
 
     def table_names
@@ -237,10 +263,16 @@ module DataDuck
 
     def self.value_to_string(value)
       string_value = ''
-      if value.respond_to? :to_s
+
+      if value.respond_to?(:strftime)
+        from_value = value.respond_to?(:utc) ? value.utc : value
+        string_value =  from_value.strftime('%Y-%m-%d %H:%M:%S')
+      elsif value.respond_to?(:to_s)
         string_value = value.to_s
       end
+
       string_value.gsub!('"', '""')
+
       return string_value
     end
   end

data/lib/dataduck/table.rb

@@ -45,7 +45,14 @@ module DataDuck
     end
 
     def actions
-      self.class.actions
+      my_actions = []
+      for_class = self.class
+      while for_class < Table
+        my_actions.concat(for_class.actions || [])
+        for_class = for_class.superclass
+      end
+
+      my_actions
     end
 
     def check_table_valid!
@@ -63,11 +70,17 @@ module DataDuck
       end
     end
 
-    def etl!(destinations)
+    def etl!(destinations, options = {})
       if destinations.length != 1
         raise ArgumentError.new("DataDuck can only etl to one destination at a time for now.")
       end
+
+      if options[:dates].nil?
+        options[:dates] = [Date.today]
+      end
+
       self.check_table_valid!
+
       destination = destinations.first
 
       if self.should_fully_reload?
@@ -77,7 +90,7 @@ module DataDuck
       batch_number = 0
       while batch_number < 1_000
         batch_number += 1
-        self.extract!(destination)
+        self.extract!(destination, options)
         self.transform!
         self.load!(destination)
 
@@ -100,7 +113,7 @@ module DataDuck
       end
     end
 
-    def extract!(destination = nil)
+    def extract!(destination = nil, options = {})
       DataDuck::Logs.info "Extracting table #{ self.name }"
 
       self.errors ||= []
@@ -109,7 +122,7 @@ module DataDuck
         source = source_spec[:source]
         my_query = self.extract_query(source_spec, destination)
         results = source.query(my_query)
-        self.data = results
+        self.data.concat(results)
       end
       self.data
     end
@@ -156,6 +169,10 @@ module DataDuck
       destination.load_table!(self)
     end
 
+    def include_with_all?
+      true
+    end
+
     def indexes
       which_columns = []
       which_columns << "id" if self.output_column_names.include?("id")
@@ -186,7 +203,7 @@ module DataDuck
     end
 
     def output_schema
-      self.class.output_schema || {}
+      self.class.output_schema || self.class.superclass.output_schema || {}
     end
 
     def output_column_names
@@ -217,8 +234,7 @@ module DataDuck
       DataDuck::Logs.info "Transforming table #{ self.name }"
 
       self.errors ||= []
-      self.class.actions ||= []
-      self.class.actions.each do |action|
+      self.actions.each do |action|
         action_type = action[0]
         action_method_name = action[1]
         if action_type == :transform
@@ -233,7 +249,16 @@ module DataDuck
     end
 
     def name
-      DataDuck::Util.camelcase_to_underscore(self.class.name)
+      fixed_name = DataDuck::Util.camelcase_to_underscore(self.class.name)
+      if fixed_name.start_with?("data_duck/")
+        fixed_name = fixed_name.split("/").last
+      end
+
+      self.prefix + fixed_name
+    end
+
+    def prefix
+      ""
     end
   end
 end

data/lib/dataduck/version.rb

@@ -2,7 +2,7 @@ module DataDuck
   if !defined?(DataDuck::VERSION)
     VERSION_MAJOR = 0
     VERSION_MINOR = 6
-    VERSION_PATCH = 2
+    VERSION_PATCH = 3
     VERSION = [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH].join('.')
   end
 end

data/lib/integrations/integration_table.rb

@@ -0,0 +1,5 @@
+module DataDuck
+  class IntegrationTable < DataDuck::Table
+    # nothing for now, but there could be integration-specific stuff here
+  end
+end

data/lib/integrations/optimizely/experiments.rb

@@ -0,0 +1,91 @@
+require_relative 'optimizely_table'
+
+require 'typhoeus'
+require 'oj'
+require 'date'
+
+module DataDuck
+  module Optimizely
+    class Experiments < DataDuck::Optimizely::OptimizelyTable
+
+      transforms :percentage_included_to_float
+      transforms :parse_datetimes
+
+      def extract!(destination, options = {})
+        self.data = []
+
+        projects_response = Typhoeus.get("https://www.optimizelyapis.com/experiment/v1/projects", headers: {'Token' => self.optimizely_api_token})
+        if projects_response.response_code != 200
+          raise Exception.new("Optimizely API for projects returned error #{ response.response_code} #{ response.body }")
+        end
+        projects = Oj.load(projects_response.body)
+
+        projects.each do |project|
+          self.extract_for_project!(project["id"])
+        end
+      end
+
+      def extract_for_project!(project_id)
+        now = DateTime.now
+
+        response = Typhoeus.get("https://www.optimizelyapis.com/experiment/v1/projects/#{ project_id }/experiments", headers: {'Token' => self.optimizely_api_token})
+
+        if response.response_code != 200
+          raise Exception.new("Optimizely API for experiments returned error #{ response.response_code} #{ response.body }")
+        end
+
+        experiments = Oj.load(response.body)
+        experiments.each do |experiment|
+          experiment[:dataduck_extracted_at] = now
+          experiment[:project_id] = project_id
+        end
+
+        self.data.concat(experiments)
+      end
+
+      def parse_datetimes(row)
+        row["created"] = DateTime.parse(row["created"])
+        row["last_modified"] = DateTime.parse(row["last_modified"])
+
+        row
+      end
+
+      def rename_description_to_name
+        row[:name] = row['description']
+
+        row
+      end
+
+      def percentage_included_to_float(row)
+        row['percentage_included'] = row['percentage_included'].to_i / 100.0
+
+        row
+      end
+
+      def indexes
+        ["id", "project_id", "primary_goal_id", "name"]
+      end
+
+      output({
+          :id => :bigint,
+          :project_id => :bigint, # integers have an overflow error because optimizely numbers get too big
+          :name => :string,
+          :shareable_results_link => :string,
+          :conditional_code => :bigtext,
+          :custom_js => :bigtext,
+          :primary_goal_id => :integer,
+          :details => :bigtext,
+          :status => :string,
+          :url_conditions => :bigtext,
+          :last_modified => :datetime,
+          :is_multivariate => :boolean,
+          :activation_mode => :string,
+          :created => :datetime,
+          :percentage_included => :float,
+          :experiment_type => :string,
+          :edit_url => :string,
+          :dataduck_extracted_at => :datetime,
+      })
+    end
+  end
+end

data/lib/integrations/optimizely/optimizely_integration.rb

@@ -0,0 +1,26 @@
+module DataDuck
+  module Optimizely
+    class OptimizelyIntegration < DataDuck::Optimizely::OptimizelyTable
+      def etl!(destinations, options = {})
+        projects = fetch_data("projects")
+        # TODO alternate way to load Optimizely data
+      end
+
+      def fetch_data(api_endpoint)
+        now = DateTime.now
+
+        response = Typhoeus.get("https://www.optimizelyapis.com/experiment/v1/#{ api_endpoint }", headers: {'Token' => self.optimizely_api_token})
+        if response.response_code != 200
+          raise Exception.new("Optimizely API for #{ api_endpoint } returned error #{ response.response_code} #{ response.body }")
+        end
+
+        rows = Oj.load(response.body)
+        rows.each do |row|
+          row[:dataduck_extracted_at] = now
+        end
+
+        rows
+      end
+    end
+  end
+end

data/lib/integrations/optimizely/optimizely_table.rb

@@ -0,0 +1,13 @@
+module DataDuck
+  module Optimizely
+    class OptimizelyTable < DataDuck::IntegrationTable
+      def optimizely_api_token
+        ENV['optimizely_api_token']
+      end
+
+      def should_fully_reload?
+        true
+      end
+    end
+  end
+end

data/lib/integrations/optimizely/projects.rb

@@ -0,0 +1,56 @@
+require_relative 'optimizely_table'
+
+require 'typhoeus'
+require 'oj'
+require 'date'
+
+module DataDuck
+  module Optimizely
+    class Projects < DataDuck::Optimizely::OptimizelyTable
+      transforms :parse_datetimes
+
+      def extract!(destination, options = {})
+        self.data = []
+
+        now = DateTime.now
+        response = Typhoeus.get("https://www.optimizelyapis.com/experiment/v1/projects", headers: {'Token' => self.optimizely_api_token})
+
+        self.data = Oj.load(response.body)
+        self.data.each do |project|
+          project[:dataduck_extracted_at] = now
+        end
+      end
+
+      def indexes
+        ["id", "account_id", "project_name"]
+      end
+
+      def parse_datetimes
+        project["created"] = DateTime.parse(project["created"])
+        project["last_modified"] = DateTime.parse(project["last_modified"])
+      end
+
+      output({
+          :id => :bigint,
+          :account_id => :bigint,
+          :code_revision => :integer,
+          :project_name => :string,
+          :project_status => :string,
+          :created => :datetime,
+          :last_modified => :datetime,
+          :library => :string,
+          :include_jquery => :bool,
+          :js_file_size => :integer,
+          :project_javascript => :bigtext,
+          :enable_force_variation => :boolean,
+          :exclude_disabled_experiments => :boolean,
+          :exclude_names => :boolean,
+          :ip_anonymization => :boolean,
+          :ip_filter => :string,
+          :socket_token => :string,
+          :dcp_service_id => :integer,
+          :dataduck_extracted_at => :datetime,
+      })
+    end
+  end
+end

data/lib/integrations/optimizely/variations.rb

@@ -0,0 +1,11 @@
+require 'typhoeus'
+
+require_relative 'optimizely_table'
+
+module DataDuck
+  module Optimizely
+    class Variations < DataDuck::Optimizely::OptimizelyTable
+      # this table should contain experiment variations and either /results or /stats for the result data
+    end
+  end
+end

data/lib/integrations/semrush/organic_results.rb

@@ -0,0 +1,59 @@
+require 'typhoeus'
+
+module DataDuck
+  module SEMRush
+    class OrganicResults < DataDuck::IntegrationTable
+      def display_limit
+        25
+      end
+
+      def key
+        ENV['semrush_api_key']
+      end
+
+      def phrases
+        raise Exception("Must implement phrases method to be an array of the phrases you want.")
+      end
+
+      def prefix
+        "semrush_"
+      end
+
+      def search_database
+        'us'
+      end
+
+      def extract!(destination, options = {})
+        dates = options[:dates]
+        if dates.nil? || dates.length == 0
+          raise Exception("Must pass at least one date.")
+        end
+
+        self.data = []
+
+        self.phrases.each do |phrase|
+          self.dates.each do |date|
+            self.extract_results_for_keyword_and_date!(phrase, date)
+          end
+        end
+      end
+
+      def extract_results_for_keyword_and_date!(phrase, date)
+        response = Typhoeus.get("http://api.semrush.com/?type=phrase_organic&key=#{ self.key }&display_limit=#{ self.display_limit }&export_columns=Dn,Ur&phrase=#{ phrase }&database=#{ self.search_database }")
+        # TODO
+      end
+
+      def indexes
+        ["date", "phrase", "domain"]
+      end
+
+      output({
+          :date => :date,
+          :phrase => :string,
+          :rank => :integer,
+          :domain => :string,
+          :url => :string,
+      })
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dataduck
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.3
 platform: ruby
 authors:
 - Jeff Pickhardt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-02 00:00:00.000000000 Z
+date: 2015-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -136,6 +136,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.12'
 - !ruby/object:Gem::Dependency
   name: sequel-redshift
   requirement: !ruby/object:Gem::Requirement
@@ -205,6 +219,13 @@ files:
 - lib/dataduck/util.rb
 - lib/dataduck/version.rb
 - lib/helpers/module_vars.rb
+- lib/integrations/integration_table.rb
+- lib/integrations/optimizely/experiments.rb
+- lib/integrations/optimizely/optimizely_integration.rb
+- lib/integrations/optimizely/optimizely_table.rb
+- lib/integrations/optimizely/projects.rb
+- lib/integrations/optimizely/variations.rb
+- lib/integrations/semrush/organic_results.rb
 - lib/templates/quickstart/table.rb.erb
 - static/logo.png
 homepage: http://dataducketl.com/