gooddata_marketo 0.0.1

data/lib/gooddata_marketo/models/child/mobj.rb

@@ -0,0 +1,68 @@
+# encoding: UTF-8
+
+class GoodDataMarketo::MObject
+
+  def initialize data, config = {}
+
+    @object = {
+        :type => data[:type],
+        :id => data[:id],
+        :raw => data
+    }
+
+    @headers = @object.keys.map{|k| k.to_s.capitalize! }
+    @headers.pop()
+
+    attributes = data[:attrib_list][:attrib]
+    attribute_map = Hash.new
+    attributes.map { |attr|
+      @headers << property = attr[:name]
+                  value = attr[:value]
+                  attribute_map[property] = value
+    }
+
+    @attributes = attribute_map
+
+  end
+
+  def type
+    @object[:type]
+  end
+
+  def to_row
+    row = [self.id,self.time,self.type,self.name]
+    @attributes.each do |attr|
+      row << attr[1]
+    end
+    row.map! { |i| i.to_s }
+  end
+
+  def headers
+    @headers
+  end
+
+  def id
+    @object[:id]
+  end
+
+  def attributes a = nil
+
+    if a
+      @attributes[a]
+    else
+      @attributes
+    end
+
+  end
+
+  def raw
+    @activity[:raw]
+  end
+
+  alias :to_a :to_row
+
+  alias :json :raw
+
+  alias :object_type :type
+
+end

data/lib/gooddata_marketo/models/etl.rb

@@ -0,0 +1,75 @@
+
+class GoodDataMarketo::ETL
+
+  def initialize config = {}
+    @queue = config[:queue]
+    @marketo = config[:marketo] || config[:client]
+  end
+
+  def determine_loads_state config = {}
+
+    loads = @marketo.loads(:user => GOODDATA_USER,
+                           :pass => GOODDATA_PASSWORD,
+                           :project => GOODDATA_PROJECT,
+                           :marketo_client => @marketo)
+
+    if loads.available?
+
+      file = loads.available.first
+      load = loads.create :name => file
+
+      load.execute
+
+      # Data from the job can now be accessed ARRAY load.storage
+      # load.storage
+
+      # Increment the load by one day if it is time related.
+
+      case load.json[:method]
+
+        when 'get_changes'
+
+          time_increment = config[:increment] || (12*60*60)
+          oca = load.arguments[:oldest_created_at]
+          lca = load.arguments[:latest_created_at]
+
+          load.arguments[:oldest_created_at] = (Time.parse(oca) + time_increment).to_s
+          load.arguments[:latest_created_at] = (Time.parse(lca) + time_increment).to_s
+
+          # If the latest time is later then today kill the load.
+          if Time.parse(lca) > Time.now
+
+            load.terminate
+
+            self.determine_loads_state
+
+            # Otherwise save the load and resume additional loads.
+          else
+
+            load.save
+
+            self.determine_loads_state
+
+          end
+
+        when 'get_multiple'
+
+          self.determine_loads_state
+
+        else
+          raise 'Unable to determine lead type ("get_multiple"/"get_changes")!'
+
+      end
+
+    else
+
+      load = @queue.pop
+      loads.create load
+
+      self.determine_loads_state
+
+    end
+
+  end
+
+end

data/lib/gooddata_marketo/models/leads.rb

@@ -0,0 +1,493 @@
+# encoding: UTF-8
+
+require 'gooddata_marketo/models/child/lead'
+require 'gooddata_marketo/models/child/activity'
+
+class GoodDataMarketo::Leads
+
+  attr_reader :client
+
+  def initialize config = {}
+
+    @client = config[:client]
+
+  end
+
+  def [](a)
+    if a.include? '@'
+      self.get_by_email(a)
+    else
+      self.get_by_id(a)
+    end
+  end
+
+  # POSSIBLE KEY TYPES FOR LEAD QUERIES
+  #
+  # IDNUM: The Marketo ID (e.g. 64)
+  # COOKIE: The value generated by the Munchkin Javascript. (e.g. id:561-HYG-937&token:_mch-marketo.com-1258067434006-50277)
+  # EMAIL: The email address associated with the lead. (e.g. rufus@marketo.com)
+  # SFDCLEADID: The lead ID from SalesForce
+  # LEADOWNEREMAIL: The Lead Owner Email
+  # SFDCACCOUNTID: The Account ID from SalesForce
+  # SFDCCONTACTID: The Contact ID from SalesForce
+  # SFDCLEADID: TheLead ID from SalesForce
+  # SFDCLEADOWNERID: The Lead owner ID from SalesForce
+  # SFDCOPPTYID: The Opportunity ID from SalesForce
+
+  def get_by_email email # http://developers.marketo.com/documentation/soap/getlead/
+    type = 'EMAIL'
+    request = { :lead_key => { :key_type => type,  :key_value => email } }
+    response = client.call(:get_lead, request)
+    # GoodDataMarketo::Lead.new response[:lead_record_list][:lead_record], :client => client (CLIENT REMOVED DUE TO PERFORMANCE ISSUE)
+    GoodDataMarketo::Lead.new response[:lead_record_list][:lead_record]
+
+  end
+
+  def get_by_id id
+    type = 'IDNUM'
+    request = { :lead_key => { :key_type => type,  :key_value => id } }
+    response = client.call(:get_lead, request)
+    # GoodDataMarketo::Lead.new response[:lead_record_list][:lead_record], :client => client (CLIENT REMOVED DUE TO PERFORMANCE ISSUE)
+    GoodDataMarketo::Lead.new response[:lead_record_list][:lead_record]
+
+  end
+
+  def get_multiple config = {} # http://developers.marketo.com/documentation/soap/getmultipleleads/
+
+    values_array = config[:ids] || config[:values]
+
+    @leads_from_call = []
+
+    # Possible types
+    # IDNUM, COOKIE, EMAIL, LEADOWNEREMAIL, SFDCACCOUNTID, SFDCCONTACTID, SFDCLEADID, SFDCLEADOWNERID, SFDCOPPTYID.
+
+    type = config[:type] || 'IDNUM'
+    xsi_type = config[:xsi_type] || 'ns1:LeadKeySelector'
+
+    if values_array.is_a? String
+      values_array = [values_array]
+    end
+
+    segments = values_array.each_slice(100).to_a
+
+    segments.each { |values|
+
+      request = {
+          :lead_selector => {
+              :key_type => type,
+              :key_values => {
+                  :string_item => values
+              }
+          },
+          :batch_size => config[:batch_size] || "100", # Unable to determine timeout rate at the 1000 so moved to 200.
+          :attributes! => { :lead_selector => { 'xsi:type' => xsi_type } }
+      }
+
+      inc = config[:include_attributes] || config[:include] || config[:types]
+      if inc
+        if inc.is_a? String
+          inc = [inc]
+        end
+        request[:activity_filter][:include_attributes] = inc
+      end
+
+      exc = config[:exclude_attributes] || config[:exclude]
+      if exc
+        if exc.is_a? String
+          exc = [exc]
+        end
+        request[:activity_filter][:exclude_attributes] = exc
+      end
+
+      activities = config[:filters] || config[:activity_name_filter] || config[:activities]
+      if activities
+        activities = [activities] if activities.is_a? String
+        request[:activity_name_filter] = Hash.new
+        request[:activity_name_filter][:string_item] = activities
+      end
+
+      # Ensure that the API call is not made with both.
+      if inc && exc
+        raise "Include and exclude attributes may not be used in the same call."
+      end
+
+      request[:start_position] = Hash.new if config[:oldest_created_at] || config[:activity_created_at] || config[:offset]
+
+      if config[:oldest_created_at]
+        begin
+          oca = StringWizard.time(config[:oldest_created_at]).to_s
+          request[:start_position][:oldest_created_at] = oca
+        rescue Exception => e
+          puts e if GoodDataMarketo.logging
+        end
+
+      end
+
+      if config[:activity_created_at]
+        begin
+          aca = StringWizard.time(config[:activity_created_at]).to_s
+          request[:start_position][:activity_created_at] = aca
+        rescue Exception => e
+          puts e if GoodDataMarketo.logging
+        end
+      end
+
+      if config[:latest_created_at]
+        begin
+          lca = StringWizard.time(config[:latest_created_at]).to_s
+          request[:start_position][:latest_created_at] = lca
+        rescue Exception => e
+          puts e if GoodDataMarketo.logging
+        end
+      end
+
+      offset_date = config[:offset]
+
+      request[:start_position][:offset] = offset_date if offset_date
+
+      # Execute a stream unless the number of leads is less than the batch limit of 100
+      begin
+
+        if values.length < 101 && values.length > 1
+
+          c = client.call(:get_multiple_leads, request)
+          c[:lead_record_list][:lead_record].each { |lead|
+            # l = GoodDataMarketo::Lead.new lead, :client => client (CLIENT REMOVED DUE TO PERFORMANCE ISSUE)
+            if client.load
+              l = lead.to_json
+            else
+              l = GoodDataMarketo::Lead.new lead
+            end
+
+            @leads_from_call << l
+          }
+
+          if client.load
+            ids = client.load.arguments[:ids].drop(values.length)
+            client.load.arguments[:ids] = ids
+            client.load.save
+
+          end
+
+
+        elsif values.length == 1
+          c = client.call(:get_multiple_leads, request)
+
+          # l = GoodDataMarketo::Lead.new c[:lead_record_list][:lead_record], :client => client (CLIENT REMOVED DUE TO PERFORMANCE ISSUE)
+          if client.load
+            l = c[:lead_record_list][:lead_record].to_json
+          else
+            l = GoodDataMarketo::Lead.new c[:lead_record_list][:lead_record]
+          end
+
+          @leads_from_call << l
+
+        else
+
+          c = client.stream(:get_multiple_leads, request)
+
+          if client.load
+
+            ids = client.load.arguments[:ids].drop(values.length)
+
+            client.load.arguments[:ids] = ids
+
+            client.load.save
+
+          end
+
+          puts "#{Time.now} => Marketo:Leads:#{c.storage.length}" if GoodDataMarketo.logging
+          c.storage.each do |request|
+            request[:lead_record_list][:lead_record].each { |lead|
+              # l = GoodDataMarketo::Lead.new lead, :client => client (CLIENT REMOVED DUE TO PERFORMANCE ISSUE)
+
+              # To conserve memory on large batches sent the raw file to load storage.
+              if client.load
+                l = lead.to_json
+              else
+                l = GoodDataMarketo::Lead.new lead
+              end
+
+              @leads_from_call << l
+            }
+          end
+
+        end
+      rescue Exception => exp
+        puts exp if GoodDataMarketo.logging
+        puts "#{Time.now} => 0 results for Marketo query."
+      end
+
+      puts "#{Time.now} => Marketo:Leads:Queue:#{@leads_from_call.length}"
+
+    }
+    client.load.log('RESPONSE') if client.load
+    client.load.storage = @leads_from_call if client.load
+    @leads_from_call
+
+  end
+
+  def get_changes(config = {}) # http://developers.marketo.com/documentation/soap/getleadchanges/
+
+    # EXAMPLE REQUEST
+    # request = {
+    #     :start_position => {
+    #         :oldest_created_at => "2013-07-01 23:58:14 -0700",
+    #     },
+    #     :activity_name_filter => {
+    #         :stringItem => ["Visit Webpage", "Click Link"] },
+    #     :batch_size => "100"
+    # }
+
+    # OPTIONAL ACTIVITIES
+    # NewLead
+    # AssocWithOpprtntyInSales
+    # DissocFromOpprtntyInSales
+    # UpdateOpprtntyInSales
+    # ChangeDataValue
+    # MergeLeads
+    # OpenEmail
+    # SendEmail
+
+    request = {
+        :start_position => Hash.new,
+        :batch_size => config[:batch_size] || '1000'
+    }
+
+    ###################
+    # Activity Config #
+    ###################
+
+    query = config[:lead] || config[:email] || config[:values] || config[:value]
+
+    if query
+      query = [query] if query.is_a? String
+      xsi_type = config[:xsi_type] || 'ns1:LeadKeySelector'
+
+      request[:lead_selector] = {}
+      request[:lead_selector][:key_values] = {}
+      request[:lead_selector][:key_type] = config[:type] || 'EMAIL'
+      request[:lead_selector][:key_values][:string_item] = query
+      request[:attributes!] = { :lead_selector => { 'xsi:type' => xsi_type } }
+
+    end
+
+    inc = config[:include_attributes] || config[:include] || config[:types]
+    if inc
+      if inc.is_a? String
+        inc = [inc]
+      end
+      request[:activity_filter][:include_attributes] = inc
+    end
+
+    exc = config[:exclude_attributes] || config[:exclude]
+    if exc
+      if exc.is_a? String
+        exc = [exc]
+      end
+      request[:activity_filter][:exclude_attributes] = exc
+    end
+
+    activities = config[:filters] || config[:activity_name_filter] || config[:activities]
+    if activities
+
+      activities = [activities] if activities.is_a? String
+      request[:activity_name_filter] = Hash.new
+      request[:activity_name_filter][:string_item] = activities
+    end
+
+    # Ensure that the API call is not made with both.
+    if inc && exc
+      raise "Include and exclude attributes may not be used in the same call."
+    end
+
+    #########################
+    # Start Position Config #
+    #########################
+
+    if config[:oldest_created_at]
+      begin
+        oca = StringWizard.time(config[:oldest_created_at])
+        request[:start_position][:oldest_created_at] = oca
+      rescue Exception => e
+        puts e if GoodDataMarketo.logging
+      end
+
+    end
+
+    if config[:latest_created_at]
+      begin
+        lca = StringWizard.time(config[:latest_created_at])
+        request[:start_position][:latest_created_at] = lca
+      rescue Exception => e
+        puts e if GoodDataMarketo.logging
+      end
+
+    end
+
+    if config[:activity_created_at]
+      begin
+        aca = StringWizard.time(config[:activity_created_at])
+        request[:start_position][:activity_created_at] = aca
+      rescue Exception => e
+        puts e if GoodDataMarketo.logging
+      end
+    end
+
+    o = config[:offset]
+    request[:start_position][:offset] = o if o
+
+    raise 'A start position type is required (:oldest_created_at, :offset, :activity_created_at)' unless oca || aca || o
+
+    c = client.stream(:get_lead_changes, request)
+
+    # Load all of the leads from the stream into an array for processes from a load or direct call.
+    # If it is a direct call, build an object, if not move the item as raw json.
+    
+    leads_from_changes_call = []
+    begin
+
+      c.storage.pmap do |request|
+
+        stored_item = request[:lead_change_record_list][:lead_change_record]
+
+        if stored_item.is_a? Array
+          stored_item.each { |activity|
+            if client.load
+              l = activity.to_json
+            else
+              l = GoodDataMarketo::Activity.new activity
+            end
+
+            leads_from_changes_call << l
+          }
+        else
+          if client.load
+            l = stored_item.to_json
+          else
+            l = GoodDataMarketo::Activity.new activity
+          end
+          leads_from_changes_call << l
+        end
+      end
+
+    rescue
+
+      puts "#{Time.now} => 0 results for Marketo query."
+    end
+
+    client.load.log('RESPONSE') if client.load
+    client.load.storage = leads_from_changes_call if client.load
+
+    leads_from_changes_call
+
+  end
+
+  def get_activities config = {} # http://developers.marketo.com/documentation/soap/getleadactivity/
+
+    # EXAMPLE HISTORY REQUEST
+    # request = {
+    #     :lead_selector => {
+    #         :key_type => "IDNUM",
+    #         :key_value => "example@host.com"},
+    #     :activity_filter => {
+    #         :include_types => {
+    #             :activity_type => ["VisitWebpage", "FillOutForm"]
+    #         }
+    #     },
+    #     :start_position => {
+    #         :"last_created_at/" => "",
+    #         :"offset/" => "" },
+    #     :batch_size => "10"
+    # }
+
+    request = {
+        :lead_key => {
+            :key_type => config[:type] || "EMAIL",
+            :key_value => config[:value]},
+        :batch_size => config[:batch_size] || "1000"
+    }
+
+    activity_types = config[:activity_types] || config[:activity_type]
+
+    if activity_types
+
+      activity_types = [activity_types] if activity_types.is_a? String
+
+      request[:activity_filter] = {}
+      request[:activity_filter][:include_types] = {}
+      request[:activity_filter][:include_types] = activity_types
+
+    end
+
+    # Check if any date options were added and if so add the date hash.
+    if config[:last_created_at] || config[:activity_created_at] || config[:latest_created_at] || config[:oldest_created_at]
+      request[:start_position] = {}
+    end
+
+    request[:start_position][:last_created_at] = StringWizard.time(config[:last_created_at]) if config[:last_created_at]
+    request[:start_position][:activity_created_at] = StringWizard.time(config[:activity_created_at]) if config[:activity_created_at]
+    request[:start_position][:latest_created_at] = StringWizard.time(config[:latest_created_at]) if config[:latest_created_at]
+    request[:start_position][:oldest_created_at] = StringWizard.time(config[:oldest_created_at]) if config[:oldest_created_at]
+
+    c = client.stream(:get_lead_activity, request)
+
+    leads_from_activities_call = []
+    begin
+      c.storage.each do |request|
+        request[:activity_record_list][:activity_record].each { |activity|
+          l = GoodDataMarketo::Activity.new activity
+          leads_from_activities_call << l
+        }
+      end
+    rescue
+      puts "#{Time.now} =>  Marketo (0) results for query. Adjust the configuration of the request or confirm there has not been changes to the structure of response from Marketo."
+    end
+
+    client.load.log('RESPONSE') if client.load
+
+    leads_from_activities_call
+
+  end
+
+  alias :get_activity :get_activities
+
+  def get_activity_by_email email
+    config = {}
+    config[:type] = 'EMAIL'
+    config[:value] = email
+
+    self.get_activities config
+
+  end
+
+  def get_activity_by_id id
+    config = {}
+    config[:type] = 'IDNUM'
+    config[:value] = id
+
+    self.get_activity config
+
+  end
+
+  def get_multiple_by_email emails, config = {}
+    config[:type] = 'EMAIL'
+    config[:ids] = emails
+    self.get_multiple config
+  end
+
+  def get_multiple_by_id ids, config = {}
+    config[:type] = 'IDNUM'
+    config[:ids] = ids
+    self.get_multiple config
+  end
+
+  def types
+    m = self.methods - Object.methods
+    m.delete(:types)
+    m
+  end
+
+end
+
+