gooddata_marketo 0.0.1-java

data/bin/process.rbx

@@ -0,0 +1,541 @@
+#!/usr/bin/ruby
+
+###################################################################
+#                                                                 #
+#                                                                 #
+# GOODDATA MARKETO CONNECTOR                                      #
+#                                                                 #
+# Ruby process for download client data from Marketo.             #
+# https://github.com/gooddata/app_store/marketo_connector         #
+#                                                                 #
+#                                                                 #
+###################################################################
+
+require 'gooddata_marketo'
+
+# Load the authorization keys.
+auth = JSON.parse(IO.read('auth.json'), :symbolize_names => true)
+
+MARKETO_SOAP_USER   = auth[:MARKETO_SOAP_USER]
+MARKETO_SOAP_KEY    = auth[:MARKETO_SOAP_KEY]
+MARKETO_REST_ID     = auth[:MARKETO_REST_ID]
+MARKETO_REST_SECRET = auth[:MARKETO_REST_SECRET]
+MARKETO_SUBDOMAIN   = auth[:MARKETO_SUBDOMAIN]
+MARKETO_API_LIMIT   = auth[:MARKETO_API_LIMIT]
+LEAD_LIST_DUMP_CSV  = auth[:LEAD_LIST_DUMP_CSV]
+GOODDATA_USER       = auth[:GOODDATA_USER]
+GOODDATA_PASSWORD   = auth[:GOODDATA_PASSWORD]
+GOODDATA_PROJECT    = auth[:GOODDATA_PROJECT]
+GOODDATA_ADS        = auth[:GOODDATA_ADS]
+
+S3_PUBLIC_KEY       = auth[:S3_PUBLIC_KEY]
+S3_PRIVATE_KEY      = auth[:S3_PRIVATE_KEY]
+S3_BUCKET           = auth[:S3_BUCKET]
+
+# No need to configure beyond this point. #
+@s3 = S3Helper.new :public_key => S3_PUBLIC_KEY,
+                   :private_key => S3_PRIVATE_KEY,
+                   :bucket => S3_BUCKET
+
+@webdav = WebDAV.new(:user => GOODDATA_USER,
+                     :pass => GOODDATA_PASSWORD,
+                     :project => GOODDATA_PROJECT)
+
+@dwh = GoodData::Datawarehouse.new(GOODDATA_USER,
+                                   GOODDATA_PASSWORD,
+                                   GOODDATA_ADS)
+
+@marketo = GoodDataMarketo.connect(:user_id => MARKETO_SOAP_USER,
+                                   :encryption_key => MARKETO_SOAP_KEY,
+                                   :api_subdomain => MARKETO_SUBDOMAIN,
+                                   :webdav => @webdav)
+
+GoodDataMarketo.logging = true
+
+# Test services
+# @marketo.test_rest
+# @marketo.test_soap
+# @s3.test
+# @dwh.test
+# @webdav.test
+
+def run_load config = {}
+
+  index = config[:index] || 1
+  @increment = config[:increment] || (12*60*60)
+  @marketo = config[:marketo_client]
+  @ads_target_table_name = config[:ads_table]
+  @lead_dump_file = "get_load_chunk_#{index}"
+
+  if @s3.exists? 'queue.json'
+
+    @queue = JSON.parse(@s3.download('queue.json'))
+    # Cancel the load if the queue is empty and delete the object.
+    if @queue.empty?
+      puts 'WARNING: Empty queue array was extracted from S3. Using queue passed in method.' if GoodDataMarketo.logging
+      @s3.delete('queue.json')
+      @queue = config[:queue]
+
+    end
+
+  else
+    @queue = config[:queue]
+  end
+
+  raise "You must pass an array of job hashs :queue_array when using run_load AND define a :marketo_client." unless @queue.length > 0 && @marketo
+  raise ":ads_table param is required with using run_load." unless @ads_target_table_name
+
+  loop do
+
+    loads = @marketo.loads(:user => GOODDATA_USER,
+                           :pass => GOODDATA_PASSWORD,
+                           :project => GOODDATA_PROJECT,
+                           :marketo_client => @marketo)
+
+    if loads.available?
+
+      file = loads.available.first
+
+      load = loads.create :name => file
+
+      @job_name = file
+      @id = load.id
+      @ads_table = load.json[:ads_table]
+      # Run the load from local or remote.
+      load.execute
+      # Data from the job can now be accessed ARRAY load.storage
+      # load.storage
+
+      if !load.storage.empty?
+
+        # Join all of the columns from the sample to all other columns.
+
+        @columns_load_aggregate = ['sys_capture_date']
+        load.storage.each { |raw_json_lead|
+
+          lead = GoodDataMarketo::Activity.new raw_json_lead if load.json[:method] == 'get_changes'
+          lead = GoodDataMarketo::Lead.new raw_json_lead if load.json[:method] == 'get_multiple'
+
+          print "\r#{Time.now} => Resolving columns: #{lead.id}\s" if GoodDataMarketo.logging
+
+          @columns_load_aggregate = @columns_load_aggregate | lead.columns
+        }
+        @columns_load_aggregate.map! { |column|
+          column.downcase.gsub('-','_')
+        }
+
+        # DEFAULTS: Use the correct ADS table.
+        if load.json[:method] == "get_multiple"
+          ads_target_table_name = 'marketo_leads'
+        elsif load.json[:method] = 'get_changes'
+          ads_target_table_name = 'marketo_changes'
+        else
+          ads_target_table_name = 'dump'
+        end
+
+        # Set up a new Table/Automatically loads current table if exists.
+
+        table = Table.new :client => @dwh, :name => @ads_table || ads_target_table_name, :columns => ['id','sys_capture_date']
+
+        if @columns_load_aggregate.length > 0
+          table.merge_columns :merge_with => @columns_load_aggregate
+
+        end
+        @csv = CSV.open("#{@id}.csv", 'w')
+
+        updated_columns = table.columns
+
+        if @columns_load_aggregate.length > 0
+          @csv << updated_columns
+        end
+
+        count = 0
+
+        ids_for_get_multiple_load = []
+
+        puts "#{Time.now} => Building objects from stream cache." if GoodDataMarketo.logging
+
+        total_leads = load.storage.length
+        total_leads_index = 0
+        load.storage.pmap do |lead|
+
+          lead = GoodDataMarketo::Activity.new lead if load.json[:method] == 'get_changes'
+          lead = GoodDataMarketo::Lead.new lead if load.json[:method] == 'get_multiple'
+
+          total_leads_index += 1
+          percentage = ((100*total_leads_index.to_f)/total_leads.to_f).round(1)
+          print "\r#{Time.now} => Transforming #{lead.id} #{percentage}%\s" if GoodDataMarketo.logging
+
+          # Get any new lead or merge lead ids and queue them for a load with get multiple.
+          ids_for_get_multiple_load << lead.values['merge_id'] if lead.values['merge_id']
+          ids_for_get_multiple_load << lead.values['lead_id'] if lead.values['lead_id']
+
+          row_to_save_csv = []
+
+          row_with_columns = updated_columns.map { |column|
+            if lead.columns.include? column
+              { column => lead.values[column] }
+            elsif lead.columns.include? "#{column}_m" # Check for anything that was removed by SQL
+              { "#{column}_m" => lead.values["#{column}_m"] }
+            elsif column == 'sys_capture_date'
+              { 'sys_capture_date' => Time.now.to_s }
+            else
+              { column => nil }
+            end
+          }
+
+          row_with_columns.each { |item|
+            c = item.to_a.flatten
+            if c[1] == nil
+              row_to_save_csv << nil
+            else
+              row_to_save_csv << c[1]
+            end
+
+          }
+
+          count += 1
+          @csv << row_to_save_csv
+
+        end
+
+        # Prepare (flush) the CSV for upload.
+        @csv.flush
+
+        table.import_csv("#{@id}.csv")
+
+        puts "#{Time.now} => #{@dwh.execute_select("SELECT id FROM #{@ads_table}").length} rows in ADS." if GoodDataMarketo.logging
+        puts "#{Time.now} => Rows extracted from CSV: #{count}" if GoodDataMarketo.logging
+        puts "#{Time.now} => New and merged ids queued: #{ids_for_get_multiple_load.length}" if GoodDataMarketo.logging
+
+        save_ids_for_get_multiple ids_for_get_multiple_load, 'a'
+
+        File.delete("#{load.json[:name]}_load.json") if File.exists? ("#{load.json[:name]}_load.json")
+        File.delete("#{@id}.csv") if File.exists? ("#{@id}.csv")
+
+        puts "#{Time.now} => Ads import of \"#{@lead_dump_file}\" complete." if GoodDataMarketo.logging
+        puts "#{Time.now} => Arguments: #{load.arguments}" if GoodDataMarketo.logging
+
+      end
+
+      case load.json[:method]
+
+        when 'get_changes'
+
+          # Increment the load by one day if it is time related.
+
+          oca = load.arguments[:oldest_created_at]
+          lca = load.arguments[:latest_created_at]
+
+          increment = Time.parse(lca) - Time.parse(oca)
+          total_time_range = Time.now - Time.parse(oca)
+
+          load.arguments[:oldest_created_at] = Time.parse(lca).to_s
+          load.arguments[:latest_created_at] = (Time.parse(lca) + increment).to_s
+
+          puts "#{Time.now} => API calls until current time: #{(total_time_range/increment).round}" if GoodDataMarketo.logging
+
+          # If the latest time is later then today kill the load.
+
+          if Time.parse(load.arguments[:latest_created_at]) > Time.now
+
+
+            load.terminate
+
+            next
+
+            # Otherwise save the load and resume additional loads.
+          else
+
+            load.save
+
+            next
+
+          end
+
+        when 'get_multiple'
+
+          determine_loads_state
+
+        else
+
+          raise 'Unable to determine lead type ("get_multiple"/"get_changes")!'
+
+          break
+
+      end
+
+    else
+
+      load = @queue.pop
+
+      puts "#{Time.now} => #{@queue.length} loads remaining."
+
+      if @queue.length > 0
+        File.open('queue.json','w'){ |f| JSON.dump(@queue, f) }
+        @s3.upload('queue.json')
+      end
+
+      if load
+
+        loads.create load
+
+        next
+
+      else
+
+        @s3.delete('queue.json')
+        File.delete('queue.json') if File.exists?('queue.json')
+
+        break
+
+      end
+
+
+    end
+
+
+  end
+
+end
+
+#####################################
+#                                   #
+#  INITIAL GET ALL MULTIPLE LEADS   #
+#                                   #
+#####################################
+# Downloads all current lead ids with REST ID.
+# Uses SOAP API to rotate through ids with get multiple.
+# Runs once, boolean value set in initial_load_get_multiple at marketo_connector_config.json
+
+def initial_load_get_multiple
+
+  @marketo.write_all_lead_ids_to_csv # Large bulk download to CSV of leads over REST API.
+
+  ids = CSV.open(LEAD_LIST_DUMP_CSV).map { |m| m[0] }
+
+  puts "#{Time.now} => #{ids.length} imported from local CSV." if GoodDataMarketo.logging
+
+  counter = 0
+
+  loop do
+
+    counter += 1
+
+    batch = ids.slice!(1..1000)
+
+    break if batch.length <= 0
+
+    get_multiple_leads_configuration = {
+        :name => "get_all_leads_chunk",
+        :type => 'leads',
+        :method => 'get_multiple',
+        :ads_table => 'marketo_leads',
+        :arguments => {
+            :ids => batch, # Notice the addition of the IDS box
+            :type => 'IDNUM'
+        }
+    }
+
+    puts "#{Time.now} => Id count in batch:#{batch.length} (Req:#{counter})" if GoodDataMarketo.logging
+
+    run_load :batch => batch,
+             :counter => counter,
+             :ads_table => 'marketo_leads',
+             :marketo_client => @marketo,
+             :queue => [get_multiple_leads_configuration]
+
+    save_ids_for_get_multiple ids, 'w'
+
+  end
+end
+
+########################################
+#                                      #
+#     INITIAL GET ALL LEAD CHANGES     #
+#                                      #
+########################################
+# Starts January 1st 2000 and increments by the day until present day.
+# All Activity Types included unless specified.
+# Runs once, boolean value set in initial_load_get_changes at marketo_connector_config.json
+
+def initial_load_get_changes
+
+  get_lead_changes_configuration = {
+      :name => 'get_lead_changes_chunk',
+      :type => 'leads',
+      :method => 'get_changes',
+      :ads_table => 'marketo_changes',
+      :arguments => {
+          # "oldest_created_at" and "latest_created_at" is the size of the increment it will to current time.
+          :oldest_created_at => 'January 1st 2004',
+          :latest_created_at => 'January 2nd 2004',
+          :filters => []
+      }
+  }
+
+  # Create a new configuration object for each activity type.
+  get_lead_changes_configuration.freeze
+
+  queue = []
+
+  @marketo.activity_types.each { |type|
+
+    g = {
+        :filters => [type]
+    }
+
+    m = get_lead_changes_configuration.dup
+    c = m.dup
+    c[:arguments] = m[:arguments].merge(g)
+
+    queue << c
+
+  }
+
+  run_load :ads_table => get_lead_changes_configuration[:ads_table],
+           :queue => queue,
+           :marketo_client => @marketo,
+           :index => 1,
+           :counter => 1
+
+  puts "#{Time.now} => Updating initial load changes to true in connector configuration." if GoodDataMarketo.logging
+
+  @s3.set_config(:initial_load_get_changes => true)
+
+end
+
+########################################
+#                                      #
+#        SYNC MULTIPLE LEADS           #
+#                                      #
+########################################
+
+def update_get_multiple_leads
+
+  file = File.open(LEAD_LIST_DUMP_CSV, 'w')
+  file.puts @s3.download(LEAD_LIST_DUMP_CSV)
+
+  ids = CSV.open(LEAD_LIST_DUMP_CSV).map { |m| m[0] }
+
+  puts "#{Time.now} => #{ids.length} imported from local CSV." if GoodDataMarketo.logging
+
+  counter = 0
+
+  loop do
+
+    counter += 1
+
+    batch = ids.slice!(1..1000)
+
+    break if batch.length <= 0
+
+    get_multiple_leads_configuration = {
+        :name => "get_all_leads_chunk",
+        :type => 'leads',
+        :method => 'get_multiple',
+        :ads_table => 'marketo_leads',
+        :arguments => {
+            :ids => batch, # Notice the addition of the IDS box
+            :type => 'IDNUM'
+        }
+    }
+
+    puts "#{Time.now} => Id count in batch:#{batch.length} (Req:#{counter})" if GoodDataMarketo.logging
+
+    run_load :batch => batch,
+             :counter => counter,
+             :ads_table => 'marketo_leads',
+             :marketo_client => @marketo,
+             :queue => [get_multiple_leads_configuration]
+
+    save_ids_for_get_multiple ids, 'w'
+
+  end
+end
+
+########################################
+#                                      #
+#          SYNC LEAD CHANGES           #
+#                                      #
+########################################
+# Synchronizes last 12 hours from process execution.
+# Runs continuously.
+
+def update_lead_changes
+
+  # Changing activity types from the default set to just Visit Webpage type.
+  #@marketo.activity_types = ['Visit Webpage']
+  #@marketo.activity_types = ['New SFDC Opportunity','Remove from Opportunity','Add to Opportunity', 'Update Opportunity']
+
+  fourty_eight_hours_ago = (Time.now - (48*60*60)).to_s
+  twenty_four_hours_ago = (Time.now - (24*60*60)).to_s
+  twelve_hours_ago = (Time.now - (12*60*60)).to_s
+  six_hours_ago = (Time.now - (6*60*60)).to_s
+
+  get_lead_changes_configuration = {
+      :name => 'get_lead_changes_chunk',
+      :type => 'leads',
+      :method => 'get_changes',
+      :ads_table => 'marketo_changes_date',
+      :arguments => {
+          # "oldest_created_at" and "latest_created_at" is the size of the increment it will to current time.
+          :oldest_created_at => twelve_hours_ago,
+          :latest_created_at => six_hours_ago,
+          :filters => []
+      }
+  }
+
+  # Create a new configuration object for each activity type.
+  get_lead_changes_configuration.freeze
+
+  queue = []
+
+  @marketo.activity_types.each { |type|
+
+    g = {
+        :filters => [type]
+    }
+
+    m = get_lead_changes_configuration.dup
+    c = m.dup
+    c[:arguments] = m[:arguments].merge(g)
+
+    queue << c
+
+  }
+
+  run_load :ads_table => get_lead_changes_configuration[:ads_table],
+           :queue => queue,
+           :marketo_client => @marketo,
+           :increment => (24*60*60),
+           :counter => 1
+
+end
+
+def save_ids_for_get_multiple ids, write_type
+  csv = CSV.open(LEAD_LIST_DUMP_CSV, write_type)
+  ids.each { |row| csv << [row] }
+  csv.flush
+  @s3.upload(LEAD_LIST_DUMP_CSV)
+end
+
+# Download the configuration file from S3. If there is not one, one will be created.
+config = @s3.get_config
+
+if !config[:initial_load_get_multiple]
+
+  initial_load_get_multiple
+
+elsif !config[:initial_load_get_changes]
+
+  initial_load_get_changes
+
+else
+
+  update_lead_changes
+
+  #Once the most recent changes downloaded, create and save a load for update_get_multiple_leads
+  update_get_multiple_leads
+
+end

data/examples/all_lead_changes.rb

@@ -0,0 +1,119 @@
+require 'gooddata_marketo'
+
+MARKETO_USER = ''
+MARKETO_KEY = ''
+MARKETO_SUBDOMAIN = ''
+
+GOODDATA_USER = ''
+GOODDATA_PASSWORD = ''
+GOODDATA_PROJECT = ''
+
+@marketo = GoodDataMarketo.connect(:user_id => MARKETO_USER, :encryption_key => MARKETO_KEY, :api_subdomain => MARKETO_SUBDOMAIN)
+
+# The specific parameters used in the method.
+
+#################
+#               #
+# CONFIGURATION #
+#               #
+#################
+
+get_lead_changes_from_march = {
+    :name => 'get_lead_changes_from_march',
+    :type => 'leads',
+    :method => 'get_changes',
+    :arguments => {
+        :oldest_created_at => 'March 26th 2014',
+        :latest_created_at => 'March 27th 2014',
+        :filters => ['Merge Leads']
+    }
+}
+
+get_multiple_from_january = {
+    :name => 'get_all_leads_january',
+    :type => 'leads',
+    :method => 'get_multiple',
+    :arguments => {
+        :ids => ['23849090','23849091'], # Notice the addition of the IDS box
+        :type => 'IDNUM'
+    }
+}
+
+@queue = [get_lead_changes_from_march, get_multiple_from_january]
+
+###################################
+#                                 #
+# DO NOT EDIT BELOW: LOAD MANAGER #
+#                                 #
+###################################
+
+
+def determine_loads_state
+
+  loads = @marketo.loads(:user => GOODDATA_USER,
+                         :pass => GOODDATA_PASSWORD,
+                         :project => GOODDATA_PROJECT,
+                         :marketo_client => @marketo)
+
+  if loads.available?
+
+    file = loads.available.first
+    load = loads.create :name => file
+
+    load.execute
+
+    # Data from the job can now be accessed ARRAY load.storage
+    # load.storage
+
+    # Increment the load by one day if it is time related.
+
+    case load.json[:method]
+
+      when 'get_changes'
+
+        one_day = (24*60*60)
+        oca = load.arguments[:oldest_created_at]
+        lca = load.arguments[:latest_created_at]
+
+        load.arguments[:oldest_created_at] = (Time.parse(oca) + one_day).to_s
+        load.arguments[:latest_created_at] = (Time.parse(lca) + one_day).to_s
+
+        # If the latest time is later then today kill the load.
+        if Time.parse(lca) > Time.now
+
+          load.terminate
+
+          determine_loads_state
+
+          # Otherwise save the load and resume additional loads.
+        else
+
+          load.save
+
+          determine_loads_state
+
+        end
+
+      when 'get_multiple'
+
+        determine_loads_state
+
+      else
+        raise 'Unable to determine lead type ("get_multiple"/"get_changes")!'
+
+    end
+
+  else
+
+    load = @queue.pop
+    loads.create load
+
+    determine_loads_state
+
+  end
+
+end
+
+# Run once, recursive until no loads are available.
+determine_loads_state
+