gooddata_marketo 0.0.1-java

data/lib/gooddata_marketo/helpers/s3.rb

@@ -0,0 +1,141 @@
+require 'uri'
+require 'net/http'
+
+class S3Helper
+
+  attr_accessor :bucket_name
+  attr_accessor :bucket
+
+  def initialize config = {}
+
+    public_key = config[:public_key]
+    private_key = config[:private_key] || config[:secret_key]
+    self.bucket_name = config[:bucket] || 'marketo_connector'
+
+    AWS.config(:access_key_id => public_key, :secret_access_key => private_key)
+
+    @s3 = AWS::S3::Client.new(region: 'us-east-1')
+
+    bucket_exists = @s3.list_buckets.include? @bucket_name
+
+    if bucket_exists == false
+      bucket = @s3.create_bucket(:bucket_name => @bucket_name)
+    else
+      bucket = bucket_exists
+    end
+
+    raise 'ERROR! :public_key, :private_key must be passed for configuration.' unless public_key && private_key
+
+  end
+
+  def test
+    begin
+      self.exists? 'tmp_dumb_file'
+      puts "#{Time.now} => SETUP: Connect to AWS S3 Bucket:#{self.bucket_name}...success!" if GoodDataMarketo.logging
+      true
+    rescue
+      false
+    end
+  end
+
+  def upload file
+    puts "#{Time.now} => Uploading:S3_Bucket#{@bucket_name}: \"#{file}\"" if GoodDataMarketo.logging
+    resp = @s3.put_object(
+        data: IO.read(file),
+        bucket_name: @bucket_name,
+        key: file
+    )
+  end
+
+  def download file
+    begin
+      puts "#{Time.now} => Downloading:S3_Bucket#{@bucket_name}: \"#{file}\"" if GoodDataMarketo.logging
+      resp = @s3.get_object(bucket_name: @bucket_name, key:file)
+      resp[:data]
+    rescue
+      nil
+    end
+  end
+
+  def get_config config = {}
+
+    file = config[:file] || 'marketo_connector_config.json'
+
+    if self.exists? file
+      json = JSON.parse(self.download(file), :symbolize_names => true)
+
+      File.open(file,'w'){ |f| JSON.dump(json, f) }
+
+      self.download(file)
+
+      json
+
+    else
+
+      json = {
+          :updated => Time.now.to_s,
+          :initial_load_get_multiple => false,
+          :initial_load_get_changes => false
+      }
+
+      File.open(file,'w'){ |f| JSON.dump(json, f) }
+
+      self.upload(file)
+
+      json
+
+    end
+
+  end
+
+  def set_config config = {}
+
+    if config[:file]
+      file = config.delete(:file)
+    else
+      file = 'marketo_connector_config.json'
+    end
+
+    if self.exists? file
+      json = JSON.parse(self.download(file), :symbolize_names => true)
+    else
+      json = Hash.new
+    end
+
+    new_json = json.merge(config)
+
+    new_json[:updated] = Time.now.to_s
+
+    File.open(file,'w'){ |f| JSON.dump(new_json, f) }
+
+    self.upload(file)
+
+    new_json
+
+  end
+
+  def latest_config
+    if File.exists?('marketo_connector_config.json')
+      File.delete('marketo_connector_config.json')
+    end
+    self.download('marketo_connector_config.json')
+  end
+
+  def exists? file
+    begin
+      @s3.get_object(bucket_name: @bucket_name, key:file)
+      true
+    rescue AWS::S3::Errors::NoSuchKey
+      false
+    end
+  end
+
+  alias :include? :exists?
+
+  def delete file
+      @s3.delete_object(bucket_name: @bucket_name, key: file)
+  end
+
+
+end
+

data/lib/gooddata_marketo/helpers/stringwizard.rb

@@ -0,0 +1,32 @@
+require 'date'
+
+module StringWizard
+  class << self
+
+    DATE_FORMATS = ['%m/%d/%Y %I:%M:%S %p', '%Y/%m/%d %H:%M:%S', '%d/%m/%Y %H:%M', '%m/%d/%Y', '%Y/%m/%d']
+
+    def time unparsed_time
+      Time.parse(unparsed_time).to_s
+      # DATE_FORMATS.each do |format|
+      #   begin
+      #     @time_string = Date.strptime(unparsed_time, format)
+      #   rescue
+      #     next
+      #   end
+      # end
+      #
+      # if @time_string.to_s.empty?
+      #   nil
+      # else
+      #   @time_string.to_s
+      # end
+
+    end
+
+    def escape_special_characters string
+      pattern = /(\'|\*|\-|\\)/
+      m = string.gsub(pattern){|match|""  + match}
+      m.gsub("\n","")
+    end
+  end
+end

data/lib/gooddata_marketo/helpers/table.rb

@@ -0,0 +1,323 @@
+require 'gooddata_marketo/data/reserved_sql_keywords'
+
+class Table
+
+  def initialize config = {}
+
+    @name_of_table = config[:table] || config[:name]
+    @inserts_to_object_count = 0
+    @checked_columns_this_insert = false
+    @dwh = config[:client]
+
+    table_exists = @dwh.table_exists? @name_of_table
+    raise 'A client is required for this module to initialize (:client => ???)' unless @dwh
+
+    # Check if the table exists, if not, create one
+
+    parsed_columns = self.columns_array_to_sql config[:columns]
+
+    begin
+      self.create_table parsed_columns
+    rescue
+      puts "#{Time.now} => #{@name_of_table} (Table) exists." if GoodDataMarketo.logging
+    end
+
+  end
+
+  def exists? table=nil
+
+    if table
+      query = table
+    else
+      query = @name_of_table
+    end
+
+    @dwh.table_exists?(query)
+
+  end
+
+  def import_csv file_path
+    puts "#{Time.now} => Loading CSV #{file_path} into ADS." if GoodDataMarketo.logging
+    begin
+      @dwh.load_data_from_csv(@name_of_table, file_path)
+      true
+    rescue Exception => exp
+      puts exp if GoodDataMarketo.logging
+    end
+
+    #@dwh.csv_to_new_table(@name_of_table, file_path)
+  end
+
+  def rename new_name
+    query = "ALTER TABLE #{@name_of_table} RENAME TO #{new_name}"
+    @dwh.execute(query)
+  end
+
+  def bulk_insert file_path
+    loc = Dir.pwd+"/"+file_path
+    query = "COPY #{@name_of_table} FROM LOCAL '#{file_path}' DELIMITER ','"
+    @dwh.execute(query)
+  end
+
+  def log message
+    puts "#{Time.now} => #{message}" if GoodDataMarketo.logging
+    message
+  end
+
+  def insert object
+
+    row = object.to_row.map {|m|
+      escaped = m.gsub("'","''")
+      m = "'#{escaped}'"
+    }
+    row[0].to_i
+    values = row.join(",")
+    columns = self.columns_array_to_string(object.headers)
+    self.log query = "INSERT INTO #{@name_of_table} (#{columns}) VALUES (#{values})"
+
+    tries = 3
+    begin
+      @dwh.execute(query)
+    rescue DataLibraryFailureException => e
+      tries -= 1
+      if tries > 0
+        sleep 3
+        retry
+      else
+        puts e if GoodDataMarketo.logging
+      end
+    end
+
+    @checked_columns_this_insert = false
+
+  end
+
+  def create_table sql_columns_string
+    sql_columns_string = "#{sql_columns_string}"
+    # EXAMPLE sql_columns_string = "id INTEGER PRIMARY KEY, name_first VARCHAR(255), name_last VARCHAR(255)) ORDER BY id SEGMENTED BY HASH(id) ALL NODES"
+    query = "CREATE TABLE #{@name_of_table} (#{sql_columns_string}) ORDER BY id SEGMENTED BY HASH(id) ALL NODES"
+
+    puts "#{Time.now} => ADS:#{query}" if GoodDataMarketo.logging
+
+    columns = sql_columns_string.split(', ')
+    columns.map! { |column|
+      s = column.split(' ')
+
+      case s[0]
+        when 'sys_capture_date' then s[1] = 'DATETIME'
+        when 'activity_date_time' then s[1] = 'DATETIME'
+        else  s[1] = 'VARCHAR(255)'
+      end
+
+      { :column_name => s[0], :data_type => s[1] }
+    }
+
+    puts "#{Time.now} => ADS:CreateTable:#{query}" if GoodDataMarketo.logging
+
+    tries = 3
+    begin
+      @dwh.create_table(@name_of_table, columns)
+
+    rescue Exception => exp
+        puts exp if GoodDataMarketo.logging
+    end
+
+  end
+
+  def add_column column, config = {}
+    # EXAMPLE column = 'column_name'
+    type = config[:type] || 'VARCHAR(255)'
+    query = "ALTER TABLE #{@name_of_table} ADD COLUMN #{column} #{type}"
+    puts "#{Time.now} => ADS:Query: #{query}"
+
+    tries = 3
+    begin
+      @dwh.execute(query)
+    rescue Exception => exp
+      tries -= 1
+      if tries > 0
+        sleep 3
+
+        retry
+      else
+        puts exp if GoodDataMarketo.logging
+      end
+    end
+
+  end
+
+  def remove_column column
+    # EXAMPLE column = 'column_name'
+    self.log query = "ALTER TABLE #{@name_of_table} DROP COLUMN #{column}"
+    tries = 3
+    begin
+      @dwh.execute(query)
+    rescue Exception => exp
+      tries -= 1
+      if tries > 0
+        sleep 3
+        retry
+      else
+        puts exp if GoodDataMarketo.logging
+      end
+    end
+  end
+
+  alias :drop_column :remove_column
+
+  def columns
+    @dwh.get_columns(@name_of_table).map { |column| column[:column_name] }
+  end
+
+  def select command
+
+    rows = []
+    @dwh.execute_select(command) do |row|
+      puts row
+      rows << row
+    end
+    self.log rows
+
+  end
+
+  alias :query :select
+
+  def columns_array_to_string columns_string_array
+
+    c = []
+    columns_string_array.each {|a| c << a }
+    id = "#{c.shift.downcase}"
+    self.log columns = c.map { |column|
+      "#{column.gsub(' ','').downcase}"
+    }
+
+    res = columns.unshift(id)
+    res.join(', ')
+
+  end
+
+  def columns_array_to_sql columns_array
+
+    c = []
+    columns_array.each {|a| c << a }
+    id = "#{c.shift.downcase} VARCHAR(255)"
+    columns = c.map { |column|
+      "#{column.gsub(' ','').downcase} VARCHAR(255)"
+    }
+
+    res = columns.unshift(id)
+    res.join(', ')
+
+  end
+
+  def export_to_csv file_path
+    tries = 3
+    begin
+      @dwh.export_to_csv @name_of_table, file_path
+    rescue Exception => exp
+      tries -= 1
+      if tries > 0
+        sleep 3
+        retry
+      else
+        puts exp if GoodDataMarketo.logging
+      end
+    end
+
+  end
+
+  def check_for_sql_parse_errors text
+
+    list = ReservedSqlKeywords.new
+    keywords = list.values
+
+    if keywords.include? text.upcase
+      puts "#{Time.now} => WARNING: Updated column key \"#{text}\" to \"#{text}\" as it is a SQL reserved keyword." if GoodDataMarketo.logging
+      text="#{text}_m"
+    else
+      text
+    end
+
+  end
+
+  def merge_columns config = {}
+
+    columns = config[:merge_with] || config[:columns]
+
+    @checked_columns_this_insert = true
+    # Set up Leads Table
+
+    # Check the table columns
+    current_columns = []
+
+    if @cached_columns
+      current_columns = @cached_columns
+    else
+
+      @dwh.get_columns(@name_of_table).each do |object|
+        current_columns << object[:column_name]
+      end
+
+      @cached_columns = current_columns
+
+    end
+
+    proposed_columns = columns.map do |column|
+      column.gsub(' ','').downcase
+    end
+
+    identical_columns = proposed_columns & current_columns == proposed_columns
+
+    if identical_columns
+
+      self.log proposed_columns
+
+    else # Find the columns that are not in the current columns and add them.
+
+      update_column_queue = Array.new
+
+      proposed_columns.pmap do |check_column|
+
+        if current_columns.find_index(check_column)
+          next # Column
+        else
+          puts "#{Time.now} => Adding new column:#{@name_of_table}:#{check_column}" if GoodDataMarketo.logging
+
+          check_column = self.check_for_sql_parse_errors(check_column)
+
+          @cached_columns << check_column # Add the column to the cache
+
+          @cached_columns.uniq!
+
+          type = 'VARCHAR(255)'
+
+          self.add_column(check_column, :type => type)
+
+        end
+
+      end
+
+      puts "#{Time.now} => Table #{@name_of_table} Merge Complete."
+
+      # If config BOOL = true also removed columns not found in the proposed columns in the warehouse
+      if config[:two_way] || config[:sync]
+        current_columns.each do |check_current_column|
+          if proposed_columns.find_index(check_current_column)
+            next # Column
+          else
+            self.log "#{Time.now} => TABLE:#{@name_of_table} + new column:#{check_current_column}"
+            self.remove_column(check_current_column)
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+  def name=
+    @name_of_table
+  end
+
+end

data/lib/gooddata_marketo/helpers/webdav.rb

@@ -0,0 +1,118 @@
+require 'uri'
+require 'net/http'
+
+class WebDAV
+
+	attr_accessor :project
+	attr_accessor :user
+	attr_accessor :password
+
+	def initialize config = {}
+
+		self.user = config[:user]
+		self.password = config[:pass] || config[:password]
+		self.project = config[:project] || config[:pid]
+
+		raise 'ERROR! :user, :password, & :project must be passed in the configuration.' unless self.user && self.project && self.password
+
+		@uri = URI.parse("https://secure-di.gooddata.com/project-uploads/")
+
+	end
+
+	def test
+		begin
+			test = self.exists? 'file'
+			puts "#{Time.now} => SETUP: Connect to GoodData WebDAV...success!" if GoodDataMarketo.logging
+			true
+		rescue
+			false
+		end
+	end
+
+	def upload file
+		http = Net::HTTP.new(@uri.host, @uri.port)
+		http.use_ssl = @uri.scheme == 'https'
+
+		request = Net::HTTP::Put.new("#{@uri.request_uri}/#{file}")
+		request.basic_auth self.user, self.password
+		request.body_stream = File.open(file)
+		request["Content-Type"] = "multipart/form-data"
+		request.add_field('Content-Length', File.size(file))
+
+		response = http.request(request)
+		if response.is_a?(Net::HTTPSuccess)
+			response
+		else
+			false
+		end
+
+	end
+
+	def download file
+		Net::HTTP.start(@uri.host, @uri.port,
+										:use_ssl => @uri.scheme == 'https') do |http|
+			request = Net::HTTP::Get.new @uri+file
+			request.basic_auth self.user, self.password
+
+			response = http.request request
+			if response.is_a?(Net::HTTPSuccess)
+				response.body
+			else
+				false
+			end
+		end
+
+	end
+
+	def get_marketo_etl_controller
+		self.download('marketo_connector.json')
+	end
+
+	def set_marketo_etl_controller
+		self.upload('marketo_connector.json')
+	end
+
+	def exists? file
+
+		Net::HTTP.start(@uri.host, @uri.port,
+										:use_ssl => @uri.scheme == 'https') do |http|
+
+			request = Net::HTTP::Get.new "#{@uri.to_s}#{file}/"
+			request.basic_auth self.user, self.password
+
+			response = http.request request
+			if response.is_a?(Net::HTTPSuccess)
+				if response.code == "200"
+					true
+				else
+					false
+				end
+			else
+				false
+			end
+		end
+
+	end
+
+	alias :include? :exists?
+
+	def delete file
+		Net::HTTP.start(@uri.host, @uri.port,
+										:use_ssl => @uri.scheme == 'https') do |http|
+
+			request = Net::HTTP::Delete.new @uri+file
+			request.basic_auth self.user, self.password
+
+			response = http.request request
+			if response.is_a?(Net::HTTPSuccess)
+				response.body
+			else
+				false
+			end
+		end
+
+	end
+
+
+end
+