gooddata_marketo 0.0.1-java

data/lib/gooddata_marketo/loads.rb

@@ -0,0 +1,235 @@
+# encoding: UTF-8
+
+# Jobs manages longer processes which may have failed or been terminated by the GoodData platform.
+
+require 'gooddata_marketo/models/load'
+require 'gooddata_marketo/helpers/webdav'
+require 'digest/sha1'
+
+class GoodDataMarketo::Loads
+
+  attr_accessor :available
+
+  def initialize config = {}
+
+    @user = config[:user]
+    @password = config[:pass] || config[:password]
+    @project = config[:project] || config[:pid]
+    @client = config[:marketo_client] || config[:marketo]
+    @webdav = WebDAV.new :user => @user, :password => @password, :project => @project
+
+  end
+
+  def instantiate config = {}
+    config[:webdav] = @webdav
+    config[:client] = @client
+    Load.new config
+  end
+
+  alias :create :instantiate
+
+  def open file
+    Load.new :file => file, :client => @client, :webdav => @webdav
+  end
+
+  def available?
+    loads = Dir.entries('.').select { |f| f.include? '_load.json' }
+    if loads.empty?
+      false
+    else
+      true
+      self.available = loads
+    end
+  end
+
+  def upload_to_webdav file
+    raise 'You must specify a :user, :password; and :project from GoodData.' unless @webdav
+    @webdav.upload file
+  end
+
+  def download_from_webdav file
+    raise 'You must specify a :user, :password; and :project from GoodData.' unless @webdav
+    @webdav.download file
+
+  end
+
+end
+
+class Load
+
+  attr_accessor :client
+  attr_accessor :storage
+  attr_accessor :arguments
+  attr_reader :json
+  attr_reader :id
+
+  def initialize config = {}
+
+    @id = Digest::SHA1.hexdigest(Time.now.to_s)
+
+    def ensure_filetype_json string
+      s = string.to_s
+      if s.include? '.json'
+        s
+      else
+        s+'_load.json'
+      end
+    end
+
+    file = config[:file] || config[:name]
+    @file = ensure_filetype_json(file)
+    @webdav = config[:webdav]
+    @client = config[:client]
+
+    # So we don't make the JSON file a mess if we merge with config.
+    config.delete(:webdav)
+    config.delete(:client)
+
+    # Check to see if the file exists locally first.
+    if File.exists? @file
+
+      @json = JSON.parse(IO.read(@file), :symbolize_names => true)
+
+    # If not is the file on WebDAV, if so, download it.
+    elsif self.on_webdav?
+
+      self.refresh
+
+    # Otherwise create a new load file locally.
+    else
+
+      config[:id] = self.id
+      File.open(@file,'w'){ |f| JSON.dump(config, f) }
+      @json = config
+
+    end
+
+    @saved = true
+    @type = @json[:type] # "leads"
+    @method = @json[:method] # "get_changes"
+    @arguments = @json[:arguments] || @json[:parameters] # ARGS = ":oldest_created_at => 'March 26th 2014', :filters => ['Merge Leads']"
+
+  end
+
+  def execute config = {}
+
+    # Assign the current load to the client.
+    client = config[:client] || @client
+    client.set_load(self)
+
+    # EXAMPLE LOADS FOR TYPE "LEADS"
+    # get_lead_changes_from_march = {
+    #     :name => 'get_lead_changes_from_march',
+    #     :type => 'leads',
+    #     :method => 'get_changes',
+    #     :arguments => {
+    #         :oldest_created_at => 'March 26th 2014',
+    #         :latest_created_at => 'March 27th 2014',
+    #         :filters => ['Merge Leads']
+    #     }
+    # }
+    #
+    # get_multiple_from_january = {
+    #     :name => 'get_all_leads_january',
+    #     :type => 'leads',
+    #     :method => 'get_multiple',
+    #     :arguments => {
+    #         :oldest_created_at => 'January 1st 2014',
+    #         :latest_created_at => 'January 2nd 2014',
+    #         :filters => ['']
+    #     }
+    # }
+
+    case @type
+
+      when "leads"
+        self.log('REQUEST')
+        client.leads.send(@method, @arguments)
+      else
+        raise 'Unable to load JOB type (Ex. "leads").'
+    end
+  end
+
+  alias :start :execute
+  alias :run :execute
+
+  # Upload the updated log to WebDAV
+  def log type
+
+    file_name = 'marketo_load_log.txt'
+    log_file = File.open(file_name,'a')
+    log_file.puts("#{self.id} --TYPE #{self.json[:type]} --METHOD #{self.json[:method]} --TIMESTAMP #{Time.now} --STATE #{type}")
+
+    self.upload(file_name)
+
+  end
+
+  # Check if the current object "@file" is on WebDAV
+  def on_webdav? f = false
+
+    if f
+      file = f
+    else
+      file = @file
+    end
+
+    if @webdav.exists? file
+      true
+    else
+      false
+    end
+
+  end
+
+  def get_argument_value key
+    @arguments[key]
+  end
+
+  def set_argument_value key, value
+    @arguments[key] = value
+  end
+
+  alias :get_argument :get_argument_value
+
+  def merge_with_load load
+    @json = load
+  end
+
+  def upload file=nil
+    if file
+      @webdav.upload file
+    else
+      @webdav.upload @file
+    end
+  end
+
+  def refresh
+    if File.exist? (@file)
+      File.delete(@file)
+    end
+    json = @webdav.download @file
+    File.open(@file,'w'){ |f| JSON.dump(json, f) }
+  end
+
+  def terminate
+    # Delete on local
+    File.delete(@file) if File.exists? @file
+
+    if @webdav.exists? @file
+      # Delete on remote
+      @webdav.delete(@file)
+    end
+  end
+
+  alias :kill :terminate
+
+  def save
+
+    File.open(@file,'w'){ |f| JSON.dump(@json, f) }
+    @saved = true
+
+    self.upload
+
+  end
+
+end

data/lib/gooddata_marketo/models/campaigns.rb

@@ -0,0 +1,57 @@
+# encoding: UTF-8
+# http://developers.marketo.com/documentation/soap/requestcampaign/
+
+class GoodDataMarketo::Campaigns
+
+  attr_reader :client
+
+  def initialize config = {}
+
+    @client = config[:client]
+
+  end
+
+  def get_campaign config = {}
+
+    request = {
+        :source => config[:source] || "MKTOWS",
+        :campaign_id => config[:id],
+        :lead_list => {
+            :lead_key => {
+                :key_type => config[:type] || "EMAIL",
+                :key_value => config[:lead] || config[:email] || config[:id] || config[:value]
+            }
+        }
+    }
+
+    client.call(:request_campaign, request)
+  end
+
+  alias :request_campaign :get_campaign
+
+  def get_campaigns_for_source config = {} # http://developers.marketo.com/documentation/soap/getcampaignsforsource/
+
+    # Ensure exact_name key is added to request if name key.
+    if config.has_key? :name
+        config[:exact_name] = "false" unless config.has_key? :exact_name
+    end
+
+    default = {
+        :source => "MKTOWS"
+        #:name => "Trigger", <-- Optional
+        #:exact_name => "false" <-- Optional
+    }
+
+    request = default.merge(config)
+
+    client.call(:get_campaigns_for_source, request)
+
+  end
+
+
+
+
+
+end
+
+

data/lib/gooddata_marketo/models/channels.rb

@@ -0,0 +1,30 @@
+class GoodDataMarketo::Client
+
+  def channels config = {} # http://developers.marketo.com/documentation/soap/getchannels/
+
+    values = config[:values] || config[:channels]
+    if values.is_a? String
+      values = [values]
+    end
+
+    request = {
+        :tag => {
+            :values => {
+                :string_item => []
+            }
+        }
+    }
+
+    if values
+      request[:tag][:values][:string_item] = values
+    end
+
+
+    self.call(:get_campaigns_for_source, request)
+
+  end
+
+
+end
+
+

data/lib/gooddata_marketo/models/child/activity.rb

@@ -0,0 +1,104 @@
+# encoding: UTF-8
+
+class GoodDataMarketo::Activity
+
+  #attr_accessor :client
+
+  def initialize data, config = {}
+
+    data = JSON.parse(data, :symbolize_names => true) unless data.is_a? Hash
+
+    @activity = {
+        :id => data[:id],
+        :activity_date_time => data[:activity_date_time].to_s, # Force this to be ADS DATETIME
+        :activity_type => data[:activity_type],
+        :mktg_asset_name => data[:mktg_asset_name],
+        :raw => data
+    }
+
+    @headers = @activity.keys.map{|k| k.to_s.capitalize! }
+    @headers.pop()
+
+    attributes = data[:activity_attributes][:attribute]
+    attribute_map = Hash.new
+    attributes.map { |attr|
+      @headers << property = attr[:attr_name].gsub(" ","_").downcase
+                  value = StringWizard.escape_special_characters(attr[:attr_value].to_s)
+                  attribute_map[property] = value
+    }
+
+    @attributes = attribute_map
+
+  end
+
+  def values
+    hash = Hash.new
+    hash['id'] = self.id
+    hash['activity_date_time'] = self.time
+    hash['activity_type'] = self.type
+    hash['mktg_asset_name'] = self.name
+    @attributes.each do |attr|
+      hash[attr[0].scan(/\w+/).join] = attr[1]
+    end
+    hash
+  end
+
+  def time
+    @activity[:activity_date_time]
+  end
+
+  def type
+    @activity[:activity_type]
+  end
+
+  def to_row
+    row = [self.id,self.time,self.type,self.name]
+    @attributes.each do |attr|
+      row << attr[1]
+    end
+    row.map! { |i| i.to_s }
+  end
+
+  alias :to_a :to_row
+
+  def headers
+    @headers.map { |h| h.scan(/\w+/).join.downcase }
+  end
+
+  alias :columns :headers
+
+  def date
+    @activity[:date]
+  end
+
+  def id
+    @activity[:id]
+  end
+
+  def name
+    @activity[:mktg_asset_name]
+  end
+
+  def attributes a = nil
+
+    if a
+      @attributes[a]
+    else
+      @attributes
+    end
+
+  end
+
+  def raw
+    @activity[:raw]
+  end
+
+  alias :to_a :to_row
+
+  alias :json :raw
+
+  alias :activity_date_time :time
+
+  alias :activity_type :type
+
+end

data/lib/gooddata_marketo/models/child/criteria.rb

@@ -0,0 +1,17 @@
+
+class GoodDataMarketo::Criteria
+  def initialize name, value, comparison = nil
+    @name = name
+    @value = value
+    @comparison = comparison
+  end
+  def data
+    obj = {}
+    o[:m_obj_criteria] = {
+        :attr_name => @name,
+        :comparison => @comparison,
+        :attr_value => @value
+    }
+    obj
+  end
+end

data/lib/gooddata_marketo/models/child/lead.rb

@@ -0,0 +1,118 @@
+# encoding: UTF-8
+
+class GoodDataMarketo::Lead
+
+  # CLIENT OBJ REMOVED DUE TO PERFORMANCE ISSUE
+  # attr_accessor :client
+
+  def initialize data, config = {}
+
+    data = JSON.parse(data, :symbolize_names => true) unless data.is_a? Hash
+
+    @lead = {
+        :id => data[:id],
+        :email => data[:email],
+        :foreign_sys_person_id => data[:foreign_sys_person_id],
+        :foreign_sys_type => data[:foreign_sys_type],
+        :raw => data
+    }
+
+    @headers = @lead.keys.map {|k| k.to_s.capitalize! }
+    @headers.pop()
+
+    attributes = data[:lead_attribute_list][:attribute]
+    attribute_map = Hash.new
+    attributes.map { |attr|
+      @headers << property = attr[:attr_name].gsub(" ","_").downcase!
+                  value = StringWizard.escape_special_characters(attr[:attr_value].to_s)
+                  attribute_map[property] = value
+    }
+
+    @attributes = attribute_map
+
+  end
+
+  def to_row
+    row = [self.id,self.email,self.foreign_sys_person_id,self.foreign_sys_type]
+    @attributes.each do |attr|
+      row << attr[1]
+    end
+    row.map! { |i| i.to_s }
+  end
+
+  def headers
+    @headers.map { |h| h.downcase }
+  end
+
+  alias :columns :headers
+
+  alias :to_a :to_row
+
+  # CLIENT OBJ REMOVED DUE TO PERFORMANCE ISSUE
+  # def get_activities config = {}
+  #   config[:type] = 'IDNUM'
+  #   config[:value] = self.id
+  #
+  #   client.leads.get_activities config
+  #
+  # end
+
+  #alias :get_activity :get_activities
+  #alias :activities :get_activities
+
+  # CLIENT OBJ REMOVED DUE TO PERFORMANCE ISSUE
+  # def get_changes config = {}
+  #   config[:type] = 'IDNUM'
+  #   config[:value] = self.id
+  #
+  #   client.leads.get_changes config
+  #
+  # end
+
+  #alias :changes :get_changes
+
+  def values
+    hash = Hash.new
+    hash['id'] = self.id
+    hash['email'] = self.email
+    hash['foreign_sys_person_id'] = self.foreign_sys_person_id
+    hash['foreign_sys_type'] = self.foreign_sys_type
+    @attributes.each do |attr|
+      hash[attr[0]] = attr[1]
+    end
+    hash
+  end
+
+  def id
+    @lead[:id]
+  end
+
+  def email
+    @lead[:email]
+  end
+
+  def foreign_sys_person_id
+    @lead[:foreign_sys_person_id]
+  end
+
+  def foreign_sys_type
+    @lead[:foreign_sys_type]
+  end
+
+  def attributes a = nil
+
+    if a
+      @attributes[a]
+    else
+      @attributes
+    end
+
+  end
+
+  def raw
+    @lead[:raw]
+  end
+
+  alias :json :raw
+
+end

data/lib/gooddata_marketo/models/child/mobj.rb

@@ -0,0 +1,68 @@
+# encoding: UTF-8
+
+class GoodDataMarketo::MObject
+
+  def initialize data, config = {}
+
+    @object = {
+        :type => data[:type],
+        :id => data[:id],
+        :raw => data
+    }
+
+    @headers = @object.keys.map{|k| k.to_s.capitalize! }
+    @headers.pop()
+
+    attributes = data[:attrib_list][:attrib]
+    attribute_map = Hash.new
+    attributes.map { |attr|
+      @headers << property = attr[:name]
+                  value = attr[:value]
+                  attribute_map[property] = value
+    }
+
+    @attributes = attribute_map
+
+  end
+
+  def type
+    @object[:type]
+  end
+
+  def to_row
+    row = [self.id,self.time,self.type,self.name]
+    @attributes.each do |attr|
+      row << attr[1]
+    end
+    row.map! { |i| i.to_s }
+  end
+
+  def headers
+    @headers
+  end
+
+  def id
+    @object[:id]
+  end
+
+  def attributes a = nil
+
+    if a
+      @attributes[a]
+    else
+      @attributes
+    end
+
+  end
+
+  def raw
+    @activity[:raw]
+  end
+
+  alias :to_a :to_row
+
+  alias :json :raw
+
+  alias :object_type :type
+
+end

data/lib/gooddata_marketo/models/etl.rb

@@ -0,0 +1,75 @@
+
+class GoodDataMarketo::ETL
+
+  def initialize config = {}
+    @queue = config[:queue]
+    @marketo = config[:marketo] || config[:client]
+  end
+
+  def determine_loads_state config = {}
+
+    loads = @marketo.loads(:user => GOODDATA_USER,
+                           :pass => GOODDATA_PASSWORD,
+                           :project => GOODDATA_PROJECT,
+                           :marketo_client => @marketo)
+
+    if loads.available?
+
+      file = loads.available.first
+      load = loads.create :name => file
+
+      load.execute
+
+      # Data from the job can now be accessed ARRAY load.storage
+      # load.storage
+
+      # Increment the load by one day if it is time related.
+
+      case load.json[:method]
+
+        when 'get_changes'
+
+          time_increment = config[:increment] || (12*60*60)
+          oca = load.arguments[:oldest_created_at]
+          lca = load.arguments[:latest_created_at]
+
+          load.arguments[:oldest_created_at] = (Time.parse(oca) + time_increment).to_s
+          load.arguments[:latest_created_at] = (Time.parse(lca) + time_increment).to_s
+
+          # If the latest time is later then today kill the load.
+          if Time.parse(lca) > Time.now
+
+            load.terminate
+
+            self.determine_loads_state
+
+            # Otherwise save the load and resume additional loads.
+          else
+
+            load.save
+
+            self.determine_loads_state
+
+          end
+
+        when 'get_multiple'
+
+          self.determine_loads_state
+
+        else
+          raise 'Unable to determine lead type ("get_multiple"/"get_changes")!'
+
+      end
+
+    else
+
+      load = @queue.pop
+      loads.create load
+
+      self.determine_loads_state
+
+    end
+
+  end
+
+end