gooddata_marketo 0.0.1-java

data/examples/all_leads.rb

@@ -0,0 +1,249 @@
+#!/usr/bin/ruby
+
+require 'gooddata_marketo'
+
+MARKETO_USER = ""
+MARKETO_KEY = ""
+MARKETO_SUBDOMAIN = ''
+
+MARKETO_AUTHORIZED_USER = ''
+MARKETO_REST_SECRET = ''
+MARKETO_REST_ID = ''
+
+MARKETO_API_LIMIT = 10000 # Total number (SOAP+REST) of API calls to make in a given day.
+LEAD_LIST_DUMP_CSV = 'marketo_leads_dump.csv' # Name of file to store all lead IDS.
+
+GOODDATA_USER = 'YOU@gooddata.com'
+GOODDATA_PASSWORD = ''
+GOODDATA_PROJECT = ''
+GOODDATA_ADS = ''
+
+@webdav = WebDAV.new(:user => GOODDATA_USER,
+                     :pass => GOODDATA_PASSWORD,
+                     :project => GOODDATA_PROJECT)
+
+@dwh = GoodData::Datawarehouse.new(GOODDATA_USER,
+                                   GOODDATA_PASSWORD,
+                                   GOODDATA_ADS)
+
+@marketo = GoodDataMarketo.connect(:user_id => MARKETO_USER,
+                                    :encryption_key => MARKETO_KEY,
+                                    :api_subdomain => MARKETO_SUBDOMAIN,
+                                    :webdav => @webdav)
+
+GoodDataMarketo.logging = true
+
+  ###############################
+  #                             #
+  #  STAGE I: GET ALL LEAD IDS  #
+  #                             #
+  ###############################
+  # REST API download of all available leads.
+
+marketo.write_all_lead_ids_to_csv :file => LEAD_LIST_DUMP_CSV
+
+ids = CSV.open(LEAD_LIST_DUMP_CSV).map { |m| m[0] }
+
+puts "#{Time.now} => #{ids.length} imported from local CSV." if GoodDataMarketo.logging
+
+def get_all_leads_batch batch, index
+
+
+  ###############################
+  #                             #
+  #  STAGE II: CONFIGURE LOADS  #
+  #                             #
+  ###############################
+  # SOAP API
+  # Configure additional loads as hashes, add them to @queue
+
+  get_all_leads = {
+      :name => "get_all_leads_chunk_#{index}",
+      :type => 'leads',
+      :method => 'get_multiple',
+      :arguments => {
+          :ids => batch, # Notice the addition of the IDS box
+          :type => 'IDNUM'
+      }
+  }
+
+  @lead_dump_file = get_all_leads[:name]
+
+  # If there are aditional loads, add to this array.
+  @queue = [get_all_leads]
+
+
+  ###############################
+  #                             #
+  #       STAGE III: LOAD       #
+  #                             #
+  ###############################
+  # SOAP API. No need to cofigure beyond this point.
+  # Loops function until no remaining jobs.
+
+  def determine_loads_state config = {}
+
+
+    loads = @marketo.loads(:user => GOODDATA_USER,
+                           :pass => GOODDATA_PASSWORD,
+                           :project => GOODDATA_PROJECT,
+                           :marketo_client => @marketo)
+
+    if loads.available?
+
+      file = loads.available.first
+      load = loads.create :name => file
+
+      # Run the load from local or remote.
+      load.execute
+
+      # Data from the job can now be accessed ARRAY load.storage
+      # load.storage
+
+      # Join all of the columns from the sample to all other columns.
+      @columns_load_aggregate = ['sys_capture_date']
+      load.storage.each { |lead| @columns_load_aggregate = @columns_load_aggregate | lead.columns }
+      @columns_load_aggregate.map! { |column|  column.downcase.gsub('-','_') }
+
+      # Set up a new Table/Automatically loads current table if exists.
+      table = Table.new :client => @dwh, :name => 'marketo_leads', :columns => ['id','email']
+
+      if @columns_load_aggregate.length > 0
+        table.merge_columns :merge_with => @columns_load_aggregate
+        @csv = CSV.open("#{@lead_dump_file}.csv", 'wb')
+      else
+        @csv = CSV.open("#{@lead_dump_file}.csv", 'wb')
+      end
+
+      updated_columns = table.columns
+
+      if @columns_load_aggregate.length > 0
+        @csv << updated_columns
+      end
+
+      count = 0
+      load.storage.each do |lead|
+
+        row_to_save_csv = []
+
+        row_with_columns = updated_columns.map { |column|
+          if lead.columns.include? column
+            { column => lead.values[column] }
+          elsif column == 'sys_capture_date'
+            { 'sys_capture_date' => Time.now.to_s }
+          else
+            { column => "NULL" }
+          end
+        }
+
+        row_with_columns.each { |item|
+          c = item.to_a.flatten
+          if c[1] == "NULL"
+            row_to_save_csv << "NULL"
+          else
+            row_to_save_csv << c[1]
+          end
+
+        }
+
+        count += 1
+        @csv << row_to_save_csv
+
+      end
+
+      # Prepare (flush) the CSV for upload.
+      @csv.flush
+
+      table.import_csv("#{@lead_dump_file}.csv")
+
+      puts "#{Time.now} => CSV:Rows:#{count}"
+      File.delete("#{@lead_dump_file}.json")
+      File.delete("#{@lead_dump_file}.csv")
+
+      puts "#{Time.now} => ADS:TableComplete:#{@lead_dump_file}" if GoodDataMarketo.logging
+
+      case load.json[:method]
+
+        when 'get_changes'
+
+          # Increment the load by one day if it is time related.
+          time_increment = (12*60*60)
+          oca = load.arguments[:oldest_created_at]
+          lca = load.arguments[:latest_created_at]
+
+          load.arguments[:oldest_created_at] = (Time.parse(oca) + time_increment).to_s
+          load.arguments[:latest_created_at] = (Time.parse(lca) + time_increment).to_s
+
+          # If the latest time is later then today kill the load.
+          if Time.parse(lca) > Time.now
+
+            load.terminate
+
+            determine_loads_state
+
+            # Otherwise save the load and resume additional loads.
+          else
+
+            load.save
+
+            determine_loads_state
+
+          end
+
+        when 'get_multiple'
+
+          determine_loads_state
+
+        else
+          raise 'Unable to determine lead type ("get_multiple"/"get_changes")!'
+
+      end
+
+    else
+
+      load = @queue.pop
+
+      if load
+        loads.create load
+        determine_loads_state
+      end
+
+    end
+
+  end
+
+# Run once, recursive until no loads are available.
+  determine_loads_state
+
+end
+
+counter = 0
+
+while ids.length > -1
+
+  counter += 1
+
+  batch = ids.slice!(1..1000)
+
+  if batch.length > 0
+    puts "#{Time.now} => Batch:Downloader:Length:#{batch.length} (Req:#{counter})" if GoodDataMarketo.logging
+    get_all_leads_batch batch, counter
+
+    CSV.open(LEAD_LIST_DUMP_CSV,'w') do |csv|
+      ids.each { |row| csv << [row] }
+    end
+
+    next
+
+  else
+
+    break
+
+  end
+
+end
+
+binding.pry
+# table.select("SELECT * FROM #{table_name} ORDER BY id")
+# SELECT * FROM marketo_leads ORDER BY id
+

data/examples/lead_changes_to_ads.rb

@@ -0,0 +1,63 @@
+# encoding: UTF-8
+# !!! JRUBY 1.7 REQUIRED !!!
+
+# MARKETO CONNECTOR EXAMPLE
+# Gets changes for Merge Leads since March 26th 2014. Creates a table if there is not already called leads in ADS. BULK INSERTS changed leads into leads table on ADS.
+
+
+require 'gooddata_marketo'
+
+MARKETO_USER = ''
+MARKETO_KEY = ''
+MARKETO_SUBDOMAIN = ''
+
+GOODDATA_USER = ''
+GOODDATA_PASSWORD = ''
+GOODDATA_PROJECT = ''
+GOODDATA_ADS  = ''
+
+#############
+#						#
+#  MARKETO  #
+#						#
+#############
+
+client = GoodDataMarketo.connect(:user_id => MARKETO_USER, :encryption_key => MARKETO_KEY, :api_subdomain => MARKETO_SUBDOMAIN)
+
+changes = client.leads.get_changes(:oldest_created_at => 'March 26th 2014', :filters => ['Merge Leads'])
+
+changed_ids = changes.map { |c| c.id }
+
+leads = client.leads.get_multiple(:ids => changed_ids, :type => 'IDNUM')
+
+CSV.open("tmp_leads_#{Process.pid}.csv", 'wb') do |csv|
+  csv << leads.first.headers
+  leads.each do |lead|
+    csv << lead.to_row
+  end
+end
+
+#############
+#						#
+#    ADS    #
+#						#
+#############
+
+dwh = GoodData::Datawarehouse.new(GOODDATA_USER, GOODDATA_PASSWORD, GOODDATA_ADS)
+
+# Set up a Table & Import BULK CSV
+table = Table.new :client => dwh, :name => 'changes'
+table.import_csv(file)
+
+# File.delete(file) # Clean up
+
+# For each change load into the leads table
+count = 0
+
+# Print the ads additions
+puts 'Print tables?'
+input = gets.chomp
+if input == "y" || input == "yes" || input == ""
+  table.select("SELECT * FROM #{table_name} ORDER BY id")
+end
+

data/gooddata_marketo.gemspec

@@ -0,0 +1,25 @@
+Gem::Specification.new do |s|
+  s.name        = 'gooddata_marketo'
+  s.version     = '0.0.1'
+  s.date        = '2015-01-01'
+  s.summary     = "Marketo SOAP/REST wrapper to CSV/GoodData ADS"
+  s.description = "A gem."
+  s.authors     = ["Patrick McConlogue"]
+  s.email       = 'patrick.mcconlogue@gooddata.com'
+  s.files       = `git ls-files`.split("\n")
+  s.homepage    = 'https://github.com/thnkr/connectors/tree/master/marketo'
+
+  s.required_ruby_version = '>= 1.9.3'
+  s.platform = 'java'
+
+  s.add_development_dependency  'aws-sdk', '~> 1.61.0'
+  s.add_development_dependency  'rubyntlm', '~> 0.3.2'
+  s.add_development_dependency  'rest-client', '~> 1.7.2'
+  s.add_development_dependency  'pmap', '~> 1.0.2'
+
+  s.add_runtime_dependency 'savon', '= 2.8.0'
+  s.add_runtime_dependency  'gooddata', '= 0.6.11'
+  s.add_runtime_dependency  'gooddata_datawarehouse', '= 0.0.5'
+
+  s.license     = 'MIT'
+end

data/lib/gooddata_marketo/adapters/rest.rb

@@ -0,0 +1,287 @@
+require 'gooddata_marketo/models/child/lead'
+
+require 'rest_client'
+require 'json'
+require 'csv'
+
+class GoodDataMarketo::RESTAdapter
+
+  attr_accessor :token
+  attr_accessor :token_timer
+
+  def initialize config = {}
+
+    @api_limit = MARKETO_API_LIMIT
+    @lead_list_dump = config[:file] || LEAD_LIST_DUMP_CSV || 'marketo_leads_dump.csv'
+    @marketo_subdomain = config[:subdomain] || MARKETO_SUBDOMAIN
+    @marketo_rest_secret = config[:secret] || config[:user] || MARKETO_REST_SECRET
+    @marketo_rest_id = config[:id] || config[:user] || MARKETO_REST_ID
+    @marketo_domain = "https://#{@marketo_subdomain}.mktorest.com"
+    @webdav = config[:webdav]
+    @lead_list_ids = []
+    @token = self.get_token unless config[:token]
+    @token_uri = "?access_token=#{@token}"
+
+    # Example endpoint = "/rest/v1/lists.json"
+
+  end
+
+  # If token has existed for more than 6000 seconds, refresh it.
+  def refresh_token?
+    if (@token_timer + 6000) < Time.now
+      self.get_token
+      true
+    else
+      false
+    end
+  end
+
+  def get url
+    @api_limit -= 1
+    begin
+      puts "#{Time.now} => REST:GET (#{url})" if GoodDataMarketo.logging
+      response = RestClient.get url
+    rescue Exception => exc
+      puts exc if GoodDataMarketo.logging
+      puts "#{Time.now} => Possible API Limit reached, last request was  ##{@api_limit} of #{MARKETO_API_LIMIT}"
+    end
+  end
+
+  def get_lead_lists next_page_token = nil
+
+    endpoint = "/rest/v1/lists.json"
+    url = @marketo_domain + endpoint + @token_uri
+
+    if next_page_token
+      endpoint = "/rest/v1/lists.json"
+      url = @marketo_domain + endpoint + @token_uri + "&nextPageToken="+next_page_token
+    end
+
+    response = self.get url
+
+    json = JSON.parse(response.body, :symbolize_names => true)
+
+    json[:result].each { |m|  @lead_list_ids << m[:id] }
+
+    next_page_token = json[:nextPageToken]
+
+    if next_page_token
+      self.get_lead_lists next_page_token
+    else
+      @lead_list_ids
+    end
+
+  end
+
+  def get_multiple_leads config = {}
+
+    self.refresh_token?
+
+    ids = config[:ids] || config[:values] || [28567,30885,32240,37161,40832]
+    domain = @marketo_domain
+    parameters = "&filterType=id"+"&filterValues="+ids.join(',')
+    endpoint= "/rest/v1/leads.json"
+    url = domain + endpoint + @token_uri + parameters
+
+    self.get url
+
+  end
+
+  def get_all_leads config = {}
+
+    self.refresh_token?
+
+    # Check if we already have a leads list, if so, download it.
+    self.load_lists
+
+    # Check if there was a stream broken while downloading a specific list.
+    self.load_last_page
+
+    # Check if there was a partial ID dump available on WebDAV, if so download it..
+    self.load_id_dump
+
+    @fields = config[:fields] || 'id'
+
+    # For each of the lead list ids, download their respective leads by id.
+    @csv = CSV.open(@lead_list_dump,'a+')
+
+    lists = @lead_list_ids
+
+    lists.each do |list|
+
+      id = @lead_list_ids.shift
+
+      domain = @marketo_domain
+      parameters = "&fields=#{@fields}"
+      endpoint= "/rest/v1/list/#{id}/leads.json"
+      url = domain + endpoint + @token_uri + parameters
+
+      # Conditional recursive function ends if nextPageToken is not available in response. Writes results to CSV.
+      def rest_lead_stream url, list_id
+
+        response = self.get url
+        json = JSON.parse(response.body, :symbolize_names => true)
+        results = json[:result]
+
+        if results
+          results.each { |result| @csv << [result[:id]] }
+          @csv.flush
+          puts "#{Time.now} => REST:leadList:#{list_id}:Results:#{results.length}" if GoodDataMarketo.logging
+        end
+
+        next_page_token = json[:nextPageToken]
+
+        # If there is another page, remember it and then attempt the next load.
+        if next_page_token
+
+          self.remember_next_page :token => token, :list => list_id
+          domain = @marketo_domain
+          parameters = "&fields=#{@fields}"
+          endpoint= "/rest/v1/list/#{list_id}/leads.json"
+          url = domain + endpoint + @token_uri + parameters + "&nextPageToken=" + next_page_token
+          rest_lead_stream url, list_id
+
+        else
+
+          # Update the local and remote lead lists
+          File.open('lead_list_ids.json','w'){ |f| JSON.dump(@lead_list_ids, f) }
+          @webdav.upload('lead_list_ids.json')
+
+        end
+
+      end
+
+      # While on this list, check to see if it failed working on it before and resume where it left off.
+      if @next_page_token_list == list
+
+        puts "#{Time.now} => REST(Resumed):leadList:#{list}:NextPageToken:#{@next_page_token}" if GoodDataMarketo.logging
+        domain = @marketo_domain
+        parameters = "" #"&fields=#{fields}"
+        endpoint= "/rest/v1/list/#{list}/leads.json"
+        url = domain + endpoint + @token_uri + parameters + "&nextPageToken=" + @next_page_token
+
+        rest_lead_stream url, list
+
+      else
+
+        rest_lead_stream url, list
+
+      end
+
+    end
+
+    # Remove lists from WebDav & Local, upload Marketo Ids Dump to webdav.
+    self.clean
+
+    # Update the etl controller that an update as occured.
+    self.resolve_with_etl_controller
+
+  end
+
+  def get_all_lead_ids config = {}
+    config[:fields] = 'id'
+    self.get_all_leads config
+  end
+
+  def get_all_lead_emails config = {}
+    config[:fields] = 'email'
+    self.get_all_leads config
+  end
+
+  def load_last_page
+
+    if @webdav.include? 'lead_list_next_page_token.json'
+      next_page_token_json = @webdav.download 'lead_list_next_page_token.json'
+      json = JSON.parse(next_page_token_json)
+    elsif File.exists? 'lead_list_next_page_token.json'
+      json = JSON.parse( IO.read('lead_list_next_page_token.json'))
+    else
+      json = {}
+    end
+
+    @next_page_token = json[:token]
+    @next_page_token_list = json[:list]
+
+  end
+
+  def load_id_dump
+    # Check if the lists leads have already been dumped. Append to it.
+    if @webdav.exists? @lead_list_dump
+      file = @webdav.download @lead_list_dump
+      puts "#{Time.now} => WEBDAV:ID_DUMP:Load" if GoodDataMarketo.logging
+      f = File.open(@lead_list_dump,'w')
+      f.write(file)
+    end
+  end
+
+  def load_lists
+    if @webdav.include? 'lead_list_ids.json'
+      lists = @webdav.download 'lead_list_ids.json'
+      @lead_list_ids = JSON.parse(lists)
+      puts "#{Time.now} => WEBDAV:ListsLoaded:#{@lead_list_ids.length}:lead_list_ids.json" if GoodDataMarketo.logging
+    else
+      self.get_lead_lists
+    end
+  end
+
+  def remember_next_page config = {}
+    File.open('lead_list_next_page_token.json','w'){ |f| JSON.dump(config, f) }
+    @webdav.upload('lead_list_next_page_token.json')
+  end
+
+  def clean
+
+    lli = 'lead_list_ids.json'
+    llnpt = 'lead_list_next_page_token.json'
+
+    # Remove list ids from webdav and local
+    if @webdav.exists?(lli)
+      @webdav.delete(lli)
+    end
+
+    if @webdav.exists?(llnpt)
+      @webdav.delete(llnpt)
+    end
+
+    File.delete(lli) if File.exists? lli
+    File.delete(llnpt) if File.exists? llnpt
+
+    # Upload marketo_dump to webdav for backup.
+    @webdav.upload(@lead_list_dump)
+
+  end
+
+  def get_token
+
+
+    domain = @marketo_domain
+    endpoint = "/identity/oauth/token?grant_type=client_credentials"
+    url = domain + endpoint + "&client_id=" + @marketo_rest_id + "&client_secret=" + @marketo_rest_secret
+    response = self.get url
+    results = JSON.parse(response.body)
+    access_token = results["access_token"]
+    @token_timer = Time.now
+    @token = access_token
+    access_token
+
+  end
+
+  def usage
+    domain = @marketo_domain
+    endpoint = "/rest/v1/stats/usage.json"
+    url = domain + endpoint + @token_uri
+
+    # Note this is the only rest call which does not use the log so as not to print the keys to log.
+    response = RestClient.get url
+
+    #Parse reponse and return only access token
+    results = JSON.parse(response.body)['result']
+  end
+
+  def resolve_with_etl_controller
+
+    puts 'Complete'
+    # Here it would update the log that a full upload as occured.
+  end
+
+end