gooddata_marketo 0.0.1

data/examples/all_lead_changes.rb

@@ -0,0 +1,119 @@
+require 'gooddata_marketo'
+
+MARKETO_USER = ''
+MARKETO_KEY = ''
+MARKETO_SUBDOMAIN = ''
+
+GOODDATA_USER = ''
+GOODDATA_PASSWORD = ''
+GOODDATA_PROJECT = ''
+
+@marketo = GoodDataMarketo.connect(:user_id => MARKETO_USER, :encryption_key => MARKETO_KEY, :api_subdomain => MARKETO_SUBDOMAIN)
+
+# The specific parameters used in the method.
+
+#################
+#               #
+# CONFIGURATION #
+#               #
+#################
+
+get_lead_changes_from_march = {
+    :name => 'get_lead_changes_from_march',
+    :type => 'leads',
+    :method => 'get_changes',
+    :arguments => {
+        :oldest_created_at => 'March 26th 2014',
+        :latest_created_at => 'March 27th 2014',
+        :filters => ['Merge Leads']
+    }
+}
+
+get_multiple_from_january = {
+    :name => 'get_all_leads_january',
+    :type => 'leads',
+    :method => 'get_multiple',
+    :arguments => {
+        :ids => ['23849090','23849091'], # Notice the addition of the IDS box
+        :type => 'IDNUM'
+    }
+}
+
+@queue = [get_lead_changes_from_march, get_multiple_from_january]
+
+###################################
+#                                 #
+# DO NOT EDIT BELOW: LOAD MANAGER #
+#                                 #
+###################################
+
+
+def determine_loads_state
+
+  loads = @marketo.loads(:user => GOODDATA_USER,
+                         :pass => GOODDATA_PASSWORD,
+                         :project => GOODDATA_PROJECT,
+                         :marketo_client => @marketo)
+
+  if loads.available?
+
+    file = loads.available.first
+    load = loads.create :name => file
+
+    load.execute
+
+    # Data from the job can now be accessed ARRAY load.storage
+    # load.storage
+
+    # Increment the load by one day if it is time related.
+
+    case load.json[:method]
+
+      when 'get_changes'
+
+        one_day = (24*60*60)
+        oca = load.arguments[:oldest_created_at]
+        lca = load.arguments[:latest_created_at]
+
+        load.arguments[:oldest_created_at] = (Time.parse(oca) + one_day).to_s
+        load.arguments[:latest_created_at] = (Time.parse(lca) + one_day).to_s
+
+        # If the latest time is later then today kill the load.
+        if Time.parse(lca) > Time.now
+
+          load.terminate
+
+          determine_loads_state
+
+          # Otherwise save the load and resume additional loads.
+        else
+
+          load.save
+
+          determine_loads_state
+
+        end
+
+      when 'get_multiple'
+
+        determine_loads_state
+
+      else
+        raise 'Unable to determine lead type ("get_multiple"/"get_changes")!'
+
+    end
+
+  else
+
+    load = @queue.pop
+    loads.create load
+
+    determine_loads_state
+
+  end
+
+end
+
+# Run once, recursive until no loads are available.
+determine_loads_state
+

data/examples/all_leads.rb

@@ -0,0 +1,249 @@
+#!/usr/bin/ruby
+
+require 'gooddata_marketo'
+
+MARKETO_USER = ""
+MARKETO_KEY = ""
+MARKETO_SUBDOMAIN = ''
+
+MARKETO_AUTHORIZED_USER = ''
+MARKETO_REST_SECRET = ''
+MARKETO_REST_ID = ''
+
+MARKETO_API_LIMIT = 10000 # Total number (SOAP+REST) of API calls to make in a given day.
+LEAD_LIST_DUMP_CSV = 'marketo_leads_dump.csv' # Name of file to store all lead IDS.
+
+GOODDATA_USER = 'YOU@gooddata.com'
+GOODDATA_PASSWORD = ''
+GOODDATA_PROJECT = ''
+GOODDATA_ADS = ''
+
+@webdav = WebDAV.new(:user => GOODDATA_USER,
+                     :pass => GOODDATA_PASSWORD,
+                     :project => GOODDATA_PROJECT)
+
+@dwh = GoodData::Datawarehouse.new(GOODDATA_USER,
+                                   GOODDATA_PASSWORD,
+                                   GOODDATA_ADS)
+
+@marketo = GoodDataMarketo.connect(:user_id => MARKETO_USER,
+                                    :encryption_key => MARKETO_KEY,
+                                    :api_subdomain => MARKETO_SUBDOMAIN,
+                                    :webdav => @webdav)
+
+GoodDataMarketo.logging = true
+
+  ###############################
+  #                             #
+  #  STAGE I: GET ALL LEAD IDS  #
+  #                             #
+  ###############################
+  # REST API download of all available leads.
+
+marketo.write_all_lead_ids_to_csv :file => LEAD_LIST_DUMP_CSV
+
+ids = CSV.open(LEAD_LIST_DUMP_CSV).map { |m| m[0] }
+
+puts "#{Time.now} => #{ids.length} imported from local CSV." if GoodDataMarketo.logging
+
+def get_all_leads_batch batch, index
+
+
+  ###############################
+  #                             #
+  #  STAGE II: CONFIGURE LOADS  #
+  #                             #
+  ###############################
+  # SOAP API
+  # Configure additional loads as hashes, add them to @queue
+
+  get_all_leads = {
+      :name => "get_all_leads_chunk_#{index}",
+      :type => 'leads',
+      :method => 'get_multiple',
+      :arguments => {
+          :ids => batch, # Notice the addition of the IDS box
+          :type => 'IDNUM'
+      }
+  }
+
+  @lead_dump_file = get_all_leads[:name]
+
+  # If there are aditional loads, add to this array.
+  @queue = [get_all_leads]
+
+
+  ###############################
+  #                             #
+  #       STAGE III: LOAD       #
+  #                             #
+  ###############################
+  # SOAP API. No need to cofigure beyond this point.
+  # Loops function until no remaining jobs.
+
+  def determine_loads_state config = {}
+
+
+    loads = @marketo.loads(:user => GOODDATA_USER,
+                           :pass => GOODDATA_PASSWORD,
+                           :project => GOODDATA_PROJECT,
+                           :marketo_client => @marketo)
+
+    if loads.available?
+
+      file = loads.available.first
+      load = loads.create :name => file
+
+      # Run the load from local or remote.
+      load.execute
+
+      # Data from the job can now be accessed ARRAY load.storage
+      # load.storage
+
+      # Join all of the columns from the sample to all other columns.
+      @columns_load_aggregate = ['sys_capture_date']
+      load.storage.each { |lead| @columns_load_aggregate = @columns_load_aggregate | lead.columns }
+      @columns_load_aggregate.map! { |column|  column.downcase.gsub('-','_') }
+
+      # Set up a new Table/Automatically loads current table if exists.
+      table = Table.new :client => @dwh, :name => 'marketo_leads', :columns => ['id','email']
+
+      if @columns_load_aggregate.length > 0
+        table.merge_columns :merge_with => @columns_load_aggregate
+        @csv = CSV.open("#{@lead_dump_file}.csv", 'wb')
+      else
+        @csv = CSV.open("#{@lead_dump_file}.csv", 'wb')
+      end
+
+      updated_columns = table.columns
+
+      if @columns_load_aggregate.length > 0
+        @csv << updated_columns
+      end
+
+      count = 0
+      load.storage.each do |lead|
+
+        row_to_save_csv = []
+
+        row_with_columns = updated_columns.map { |column|
+          if lead.columns.include? column
+            { column => lead.values[column] }
+          elsif column == 'sys_capture_date'
+            { 'sys_capture_date' => Time.now.to_s }
+          else
+            { column => "NULL" }
+          end
+        }
+
+        row_with_columns.each { |item|
+          c = item.to_a.flatten
+          if c[1] == "NULL"
+            row_to_save_csv << "NULL"
+          else
+            row_to_save_csv << c[1]
+          end
+
+        }
+
+        count += 1
+        @csv << row_to_save_csv
+
+      end
+
+      # Prepare (flush) the CSV for upload.
+      @csv.flush
+
+      table.import_csv("#{@lead_dump_file}.csv")
+
+      puts "#{Time.now} => CSV:Rows:#{count}"
+      File.delete("#{@lead_dump_file}.json")
+      File.delete("#{@lead_dump_file}.csv")
+
+      puts "#{Time.now} => ADS:TableComplete:#{@lead_dump_file}" if GoodDataMarketo.logging
+
+      case load.json[:method]
+
+        when 'get_changes'
+
+          # Increment the load by one day if it is time related.
+          time_increment = (12*60*60)
+          oca = load.arguments[:oldest_created_at]
+          lca = load.arguments[:latest_created_at]
+
+          load.arguments[:oldest_created_at] = (Time.parse(oca) + time_increment).to_s
+          load.arguments[:latest_created_at] = (Time.parse(lca) + time_increment).to_s
+
+          # If the latest time is later then today kill the load.
+          if Time.parse(lca) > Time.now
+
+            load.terminate
+
+            determine_loads_state
+
+            # Otherwise save the load and resume additional loads.
+          else
+
+            load.save
+
+            determine_loads_state
+
+          end
+
+        when 'get_multiple'
+
+          determine_loads_state
+
+        else
+          raise 'Unable to determine lead type ("get_multiple"/"get_changes")!'
+
+      end
+
+    else
+
+      load = @queue.pop
+
+      if load
+        loads.create load
+        determine_loads_state
+      end
+
+    end
+
+  end
+
+# Run once, recursive until no loads are available.
+  determine_loads_state
+
+end
+
+counter = 0
+
+while ids.length > -1
+
+  counter += 1
+
+  batch = ids.slice!(1..1000)
+
+  if batch.length > 0
+    puts "#{Time.now} => Batch:Downloader:Length:#{batch.length} (Req:#{counter})" if GoodDataMarketo.logging
+    get_all_leads_batch batch, counter
+
+    CSV.open(LEAD_LIST_DUMP_CSV,'w') do |csv|
+      ids.each { |row| csv << [row] }
+    end
+
+    next
+
+  else
+
+    break
+
+  end
+
+end
+
+binding.pry
+# table.select("SELECT * FROM #{table_name} ORDER BY id")
+# SELECT * FROM marketo_leads ORDER BY id
+

data/examples/lead_changes_to_ads.rb

@@ -0,0 +1,63 @@
+# encoding: UTF-8
+# !!! JRUBY 1.7 REQUIRED !!!
+
+# MARKETO CONNECTOR EXAMPLE
+# Gets changes for Merge Leads since March 26th 2014. Creates a table if there is not already called leads in ADS. BULK INSERTS changed leads into leads table on ADS.
+
+
+require 'gooddata_marketo'
+
+MARKETO_USER = ''
+MARKETO_KEY = ''
+MARKETO_SUBDOMAIN = ''
+
+GOODDATA_USER = ''
+GOODDATA_PASSWORD = ''
+GOODDATA_PROJECT = ''
+GOODDATA_ADS  = ''
+
+#############
+#						#
+#  MARKETO  #
+#						#
+#############
+
+client = GoodDataMarketo.connect(:user_id => MARKETO_USER, :encryption_key => MARKETO_KEY, :api_subdomain => MARKETO_SUBDOMAIN)
+
+changes = client.leads.get_changes(:oldest_created_at => 'March 26th 2014', :filters => ['Merge Leads'])
+
+changed_ids = changes.map { |c| c.id }
+
+leads = client.leads.get_multiple(:ids => changed_ids, :type => 'IDNUM')
+
+CSV.open("tmp_leads_#{Process.pid}.csv", 'wb') do |csv|
+  csv << leads.first.headers
+  leads.each do |lead|
+    csv << lead.to_row
+  end
+end
+
+#############
+#						#
+#    ADS    #
+#						#
+#############
+
+dwh = GoodData::Datawarehouse.new(GOODDATA_USER, GOODDATA_PASSWORD, GOODDATA_ADS)
+
+# Set up a Table & Import BULK CSV
+table = Table.new :client => dwh, :name => 'changes'
+table.import_csv(file)
+
+# File.delete(file) # Clean up
+
+# For each change load into the leads table
+count = 0
+
+# Print the ads additions
+puts 'Print tables?'
+input = gets.chomp
+if input == "y" || input == "yes" || input == ""
+  table.select("SELECT * FROM #{table_name} ORDER BY id")
+end
+

data/gooddata_marketo.gemspec

@@ -0,0 +1,24 @@
+Gem::Specification.new do |s|
+  s.name        = 'gooddata_marketo'
+  s.version     = '0.0.1'
+  s.date        = '2015-01-01'
+  s.summary     = "Marketo SOAP/REST wrapper to CSV/GoodData ADS"
+  s.description = "A gem."
+  s.authors     = ["Patrick McConlogue"]
+  s.email       = 'patrick.mcconlogue@gooddata.com'
+  s.files       = `git ls-files`.split("\n")
+  s.homepage    = 'https://github.com/thnkr/connectors/tree/master/marketo'
+
+  s.required_ruby_version = '>= 1.9.3'
+
+  s.add_development_dependency  'aws-sdk', '~> 1.61.0'
+  s.add_development_dependency  'rubyntlm', '~> 0.3.2'
+  s.add_development_dependency  'rest-client', '~> 1.7.2'
+  s.add_development_dependency  'pmap', '~> 1.0.2'
+
+  s.add_runtime_dependency 'savon', '= 2.8.0'
+  s.add_runtime_dependency  'gooddata', '= 0.6.11'
+  s.add_runtime_dependency  'gooddata_datawarehouse', '= 0.0.5'
+
+  s.license     = 'MIT'
+end