gooddata_marketo 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cac617b71a6f470a15bb65b9c9f95335c3024546
+  data.tar.gz: 1dfe0328779ef33026080e6af6ec7609421e0e0f
+SHA512:
+  metadata.gz: fca9b55c97b89799b048337ac0e10e161ab47f21c822ee5d5b610a77cbdaff50c28b7389211559a54796d130e7dd291501d129a7246fd556f7387c3600c635c7
+  data.tar.gz: 60cd618ed5620155e2211985e045d0967395f4a72e5b5802603b36f40f49272416b7fe5f8eada927e5afab1aa0b6f33bc840599fa6737675b59f299876fa1683

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+gem 'savon', '2.8.0'
+gem 'rubyntlm', '0.3.2'
+gem 'gooddata', '0.6.11'
+gem 'rest-client', '1.7.2'
+gem 'aws-sdk', '1.61.0'
+gem 'pmap', '1.0.2'
+gem 'gooddata_datawarehouse', '0.0.5'

data/Gemfile.lock

@@ -0,0 +1,131 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.0)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    akami (1.2.2)
+      gyoku (>= 0.4.0)
+      nokogiri
+    aws-sdk (1.61.0)
+      aws-sdk-v1 (= 1.61.0)
+    aws-sdk-v1 (1.61.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+    builder (3.2.2)
+    coderay (1.1.0)
+    docile (1.1.5)
+    erubis (2.7.0)
+    faraday (0.9.1)
+      multipart-post (>= 1.2, < 3)
+    faraday_middleware (0.9.1)
+      faraday (>= 0.7.4, < 0.10)
+    ffi (1.9.6-java)
+    gli (2.12.2)
+    gooddata (0.6.11)
+      activesupport (~> 4.1, >= 4.1.0)
+      aws-sdk (~> 1.45)
+      bundler (~> 1.7, >= 1.7.3)
+      docile (~> 1.1, >= 1.1.5)
+      erubis (~> 2.7, >= 2.7.0)
+      gli (~> 2.12, >= 2.12.2)
+      highline (~> 1.6, >= 1.6.21)
+      i18n (~> 0.6, >= 0.6.9)
+      json_pure (~> 1.8, >= 1.8.1)
+      multi_json (~> 1.10, >= 1.10.0)
+      parseconfig (~> 1.0, >= 1.0.4)
+      pmap (~> 1.0, >= 1.0.1)
+      pry (~> 0.9.12.6)
+      rest-client (~> 1.7, >= 1.7.2)
+      restforce (~> 1.5, >= 1.5.0)
+      rubyzip (~> 1.1, >= 1.1.0)
+      salesforce_bulk_query (~> 0.0)
+      terminal-table (~> 1.4, >= 1.4.5)
+    gyoku (1.2.2)
+      builder (>= 2.1.2)
+    hashie (3.3.2)
+    highline (1.6.21)
+    httpi (2.3.0)
+      rack
+    i18n (0.7.0)
+    json (1.8.2)
+    json (1.8.2-java)
+    json_pure (1.8.2)
+    macaddr (1.7.1)
+      systemu (~> 2.6.2)
+    method_source (0.8.2)
+    mime-types (1.25.1)
+    mini_portile (0.6.2)
+    minitest (5.5.0)
+    multi_json (1.10.1)
+    multipart-post (2.0.0)
+    netrc (0.10.2)
+    nokogiri (1.6.5)
+      mini_portile (~> 0.6.0)
+    nokogiri (1.6.5-java)
+    nori (2.4.0)
+    parseconfig (1.0.6)
+    pmap (1.0.2)
+    pry (0.9.12.6)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    pry (0.9.12.6-java)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+      spoon (~> 0.0)
+    rack (1.6.0)
+    rest-client (1.7.2)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    restforce (1.5.1)
+      faraday (~> 0.9.0)
+      faraday_middleware (>= 0.8.8)
+      hashie (>= 1.2.0, < 4.0)
+      json (>= 1.7.5, < 1.9.0)
+    rubyntlm (0.3.2)
+    rubyzip (1.1.6)
+    salesforce_bulk_query (0.0.6)
+      json (~> 1.8)
+      xml-simple (~> 1.1)
+    savon (2.8.0)
+      akami (~> 1.2)
+      builder (>= 2.1.2)
+      gyoku (~> 1.2)
+      httpi (~> 2.3)
+      nokogiri (>= 1.4.0)
+      nori (~> 2.4)
+      uuid (~> 2.3.7)
+      wasabi (= 3.3.0)
+    slop (3.6.0)
+    spoon (0.0.4)
+      ffi
+    systemu (2.6.4)
+    terminal-table (1.4.5)
+    thread_safe (0.3.4)
+    thread_safe (0.3.4-java)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    uuid (2.3.7)
+      macaddr (~> 1.0)
+    wasabi (3.3.0)
+      httpi (~> 2.0)
+      mime-types (< 2.0.0)
+      nokogiri (>= 1.4.0)
+    xml-simple (1.1.4)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  aws-sdk (= 1.61.0)
+  gooddata (= 0.6.11)
+  pmap (= 1.0.2)
+  rest-client (= 1.7.2)
+  rubyntlm (= 0.3.2)
+  savon (= 2.8.0)

data/README.md

@@ -0,0 +1,207 @@
+GOODDATA MARKETO CONNECTOR
+==========================
+Marketo SOAP/REST services to GoodData ADS
+
+## Overview
+With the Marketo Connector a complete transfer of all available lead data is automatically moved *and* kept in sync with GoodData ADS. Supporting both the Marketo REST & SOAP the GoodData Marketo Connector adapts to the source data making installation as simple as entering your login information.
+
+### Marketo
+The central concept of the Marketo API is a "Lead". Each lead contains default attributes like City or Email in addition to custom attributes added by the client. The Marketo Connector downloads and synchronizes all leads and lead changes for a given Marketo client. The connector utitlizes two main calls, *get_lead_changes* and *get_multiple_leads*, these feed into the ADS tables *marketo_changes* and *marketo_leads* respectively. For examples, take a look at the code within the *examples* directory.
+
+### Getting Started
+
+1. Install jruby `rvm install jruby` and then `rvm use jruby` (tested on jruby 1.7.16.1)
+2. [Install DSS for Ruby](https://confluence.intgdc.com/pages/viewpage.action?title=DSS+Alpha+Version+-+User+Guide&spaceKey=plat)
+3. Clone this repository.
+4. Configure authentication in bin/auth.json.
+5. Zip the the entire "bin" folder to a new zip file.
+6. Upload the zip file it as a Ruby process, schedule it to run daily, every 12 hours.
+
+### Configuration & Keys
+To use the Marketo Connector you must have credentials for the following services.
+
+- [Marketo REST API](http://developers.marketo.com/blog/quick-start-guide-for-marketo-rest-api/).
+- [Marketo SOAP API](http://developers.marketo.com/documentation/soap/).
+- [GoodData Username, Password, & Project ID](https://na1.secure.gooddata.com/account.html?clicked_trial_link=1#/registration/projectTemplate/urn%3Agooddata%3AOnboarding).
+- [AWS S3](http://aws.amazon.com/s3/).
+
+#### File Tree
+
+    ├── bin
+    │   ├── Gemfile
+    │   ├── auth.json
+    │   ├── main.rb
+    │   └── process.rbx
+    │
+    ├── examples
+    │   ├── all_lead_changes.rb
+    │   ├── all_leads.rb
+    │   └── lead_changes_to_ads.rb
+    │
+    ├── lib
+    │   ├── gooddata_marketo
+    │   │   ├── adapters
+    │   │   │   └── rest.rb
+    │   │   ├── client.rb
+    │   │   ├── data
+    │   │   │   ├── activity_types.rb
+    │   │   │   └── reserved_sql_keywords.rb
+    │   │   ├── helpers
+    │   │   │   ├── s3.rb
+    │   │   │   ├── stringwizard.rb
+    │   │   │   ├── table.rb
+    │   │   │   └── webdav.rb
+    │   │   ├── loads.rb
+    │   │   └── models
+    │   │       ├── campaigns.rb
+    │   │       ├── channels.rb
+    │   │       ├── child
+    │   │       │   ├── activity.rb
+    │   │       │   ├── criteria.rb
+    │   │       │   ├── lead.rb
+    │   │       │   └── mobj.rb
+    │   │       ├── etl.rb
+    │   │       ├── leads.rb
+    │   │       ├── load.rb
+    │   │       ├── mobjects.rb
+    │   │       ├── streams.rb
+    │   │       ├── tags.rb
+    │   │       └── validate.rb
+    │   └── gooddata_marketo.rb
+    │
+    └── tests
+        └── test.rb
+
+## Custom Scripts
+In addition to running the main process you can also write scripts for exploration and small transformations with the Marketo API. To start, require the Marketo Connector and create a client.
+
+    require 'marketo_connector'
+
+    user = ""
+    key = ""
+    subdomain = ""
+
+    client = GoodDataMarketo.connect(:user_id => user,
+                                      :encryption_key => key,
+                                      :api_subdomain => subdomain)
+
+From the client you can get a specific lead or a stream of ids.
+
+    lead = client.leads.get_by_email('email@email.com')
+    leads = client.leads.get_multiple ['23590','2930','9209'], :filters => 'Merge Leads', :type => 'IDNUM'
+
+## Customizing Calls
+Marketo API queries can be very time consuming, in the hours or days range, before a response returns. This is why the Marketo API allows for very complex filtration options like creation date, data-type, request size etc. Filtering your queries by some level is **required** due to the size and slower speed of Marketo. Keep in mind, as of this post RUBY Processes on the GoodData platform automatically terminate after 5 hours. Setting up configuration options for the two main calls, `client.leads.get_changes` and `client.leads.get_multiple` is almost identical, you can learn more about them [here](http://developers.marketo.com/documentation/soap/getmultipleleads/).
+
+#### Important Configurations
+
+* ` :timeout => Int `
+
+Specifies the amount of time before you want a call to the API to fail. Currently the library wide default is 120 which is also the the time allocated to the SAVON GEM in the initialization of the client. To go past that you will need to change the timeout when the client is first initialized.
+
+* ` :filters => Array `
+
+A list of activity types which are translated into the *activity_name_filters*. Due to changes in the Marketo API becareful to print these in complete form like 'Merge Leads' instead of 'MergeLeads' or 'Webpage Clicks' instead of 'webpageClicks'.
+
+* ` :type => String `
+
+Defines the type of your query. For instance, `:type => 'IDNUM'` would mean you are searching for data using a Marketo ID, or you might use `:type => 'EMAIL'` if you are querying using an email. In specific cases like `client.leads.get_by_id` the type is set for you.
+
+If the given call you are looking for is not fully implemented, you can always build the complete soap request your self by calling `client.call` directly.
+Outside of the authentication options, the default configuration for the client is in *marketo_connector/client.rb* under the parameter *DEFAULT_CONFIG*.
+
+## Leads
+All calls to Marketo return objects that contain attributes. Currently the Marketo Connector supports *Lead* and *Activity* where Activity is a child of lead containing the ID and values pertaining to the lead's activity. Custom attributes can be added to leads which include values, accessing them looks like this:
+
+    leads = client.leads.get_changes :last_created_at => '12/02/2013'
+    lead = leads.first
+    custom_form = lead.attributes['c__Custom_Form']
+
+Where `custom_form` will now contain the given value of the lead. Lead and Activity objects can also be flattened to save to CSV for parsing in the future.
+
+    lead = client.leads.get_by_email('john@smith.com')
+
+    CSV.open('leads.csv', 'wb') do |csv|
+
+      csv << lead.headers # => Incase you wanted to wanted the headers of attributes also saved to CSV.
+      csv << lead.to_row
+
+    end
+
+You can also extract the activities of a given lead through `lead.activities` or changes of a given lead with `lead.changes`. For example if you wanted to write the attributes for leads which included the City 'San Francisco' to csv...
+
+    leads = client.leads.get_changes :filters => ['Email Opens', 'Merge Leads'], :lastest_created_at => '10/11/2014'
+
+    san_francisco_leads = leads.select {|lead| lead.attributes['City'] == 'San Francisco'}
+
+    CSV.open('leads.csv', 'wb') do |csv|
+      san_francisco_leads.each do |lead|
+        csv << lead.to_row
+      end
+    end
+
+In addition, you can always work with the raw response of the lead object with `lead.raw`.
+
+## Streams
+Specific to getMultipleLeads, getLeadChanges, getMObjects, and getLeadActivity calls, the Marketo SOAP API supports "Streams" which allow you to increment through large responses by offset stamp or stream ID (just like pagenate).
+Within the client script (*lib/marketo_connector/client.rb*) are two calls, a streaming function which automatically works through APIs, building a complete object.
+
+In the event the Marketo times out **AND** you gave specific date configurations like *:started_at => TIME* or *:oldest_created_at => TIME*, the stream will increment your calls into smaller time windows until it does receive a response within the given timeout configuration. This is helpful to insure that the most relevant data is always in a clients account. You can also deactivate this by passing the configuration symbol *:safe* and setting it to **false**.
+
+## Custom Objects
+Marketo allows you to query the API using what it calls *criteria* which is a hash comprised of the attribute name and value with a comparison statement like *EQ* "Equals" or *LT* "Less Then". Many criteria hashes can be sent with a given request, due to the size of these queries it is also suggest that you do so but rather than forcing you to do with large arrays of criteria hashes you simply add criteria to the given *client.mobject* like so:
+
+    criteria_one = {
+        :attr_name => "Id", # See the types of content it can search above.
+        :comparison => "LE",
+        :attr_value => "1010"
+    }
+
+    >> client.mobjects.add(criteria_one)
+    [{:attr_name => 'Id', :comparison => 'LE', :attr_value => '1010'}]
+
+    criteria_two = {
+        :attr_name => "Zip", # See the types of content it can search above.
+        :comparison => "EQ",
+        :attr_value => "94104"
+    }
+
+    >> client.mobjects.add(criteria_two)
+    [{:attr_name => 'Id', :comparison => 'LE', :attr_value => '1010'}, {:attr_name => 'Zip', :comparison => 'EQ', :attr_value => '94104'}]
+
+The data is stored in a class array which allows you to do this:
+
+    objects = client.mobjects.get
+
+You can also make requests with one criteria hash like this:
+
+    client.mobjects.get :criteria => HASH
+
+Finally criteria can be removed by name with: ` client.mbobjects.remove_criteria 'NAME' `.
+
+## Supported Calls
+
+* [` client.leads.get_changes `](http://developers.marketo.com/documentation/soap/getleadchanges/) - Optional config. Streaming is required.
+
+* [` client.leads.get_multiple `](http://developers.marketo.com/documentation/soap/getmultipleleads/) - Requires an query Array, optional Config.
+
+* [` client.leads.get_lead `](http://developers.marketo.com/documentation/soap/getlead/) - Requires query String, optional config.
+
+* [` client.mobjects `](http://developers.marketo.com/documentation/soap/getcustomobjects/) - Requires criteria to be set (See Custom Objects above)
+
+* [` client.usage `](http://developers.marketo.com/documentation/rest/get-daily-usage/) -  Requires API key to within REST config.
+
+* [` client.channels `](http://developers.marketo.com/documentation/soap/getchannels/) - Requires query value, optional config.
+
+* [` client.tags `](http://developers.marketo.com/documentation/soap/gettags/) - Requires the Lead email or ID for the query, optional config.
+
+## Custom Requests
+You can call custom requests should you need want more specific control or use of a feature.
+
+#### Direct Call
+A single call is ` client.call(WEB_METHOD, MESSAGE) ` where WEB_METHOD is the specific SOAP function you are attempting to use and message is the SOAP request message (as a hash).
+
+#### Streaming Calls
+In addition for web methods that support streaming, you can using `client.stream(WEB_METHOD, MESSAGE)` to automatically iterate through either *offset* or *start_position* and return an array of results in the correct type.
+
+

data/bin/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+
+gem 'savon', '2.8.0'
+gem 'rubyntlm', '0.3.2'
+gem 'gooddata', '0.6.11'
+gem 'rest-client', '1.7.2'
+gem 'aws-sdk', '1.61.0'
+gem 'pmap', '1.0.2'
+gem 'gooddata_datawarehouse', '0.0.5'
+gem 'gooddata_marketo', '0.0.1'

data/bin/auth.json

@@ -0,0 +1,17 @@
+{
+  "MARKETO_SOAP_USER" : "lionbridge1_954603944F16DBA4B8DCE3",
+  "MARKETO_SOAP_KEY" : "248763534270886944FF1166DDBBAA56BB9915989447",
+  "MARKETO_REST_ID" : "Ayc9WFfqN1SHrDNVTcw3PoaulwBomGSr",
+  "MARKETO_REST_SECRET" : "2581e771-9030-4ecd-8960-9a16759166d2",
+  "MARKETO_SUBDOMAIN" : "972-PWS-816",
+  "MARKETO_API_LIMIT" : 10000,
+  "LEAD_LIST_DUMP_CSV" : "marketo_lead_dump.csv",
+  "GOODDATA_USER" : "patrick.mcconlogue@gooddata.com",
+  "GOODDATA_PASSWORD" : "dxdemo1",
+  "GOODDATA_PROJECT" : "wj8uljb34dmz0mp92dsypuksv2j8xw5m",
+  "GOODDATA_ADS" : "t4d9806bbea8656285664cf0160d0f07",
+  "S3_PUBLIC_KEY" : "AKIAISTOSUAVASUBEUSQ",
+  "S3_PRIVATE_KEY" : "KbV1sAkB3EhgLoljUs8V21SLYF0RALPgaNZb5W+P",
+  "S3_BUCKET" : "marketo_connector_lionbridge"
+}
+

data/bin/process.rbx

@@ -0,0 +1,541 @@
+#!/usr/bin/ruby
+
+###################################################################
+#                                                                 #
+#                                                                 #
+# GOODDATA MARKETO CONNECTOR                                      #
+#                                                                 #
+# Ruby process for download client data from Marketo.             #
+# https://github.com/gooddata/app_store/marketo_connector         #
+#                                                                 #
+#                                                                 #
+###################################################################
+
+require 'gooddata_marketo'
+
+# Load the authorization keys.
+auth = JSON.parse(IO.read('auth.json'), :symbolize_names => true)
+
+MARKETO_SOAP_USER   = auth[:MARKETO_SOAP_USER]
+MARKETO_SOAP_KEY    = auth[:MARKETO_SOAP_KEY]
+MARKETO_REST_ID     = auth[:MARKETO_REST_ID]
+MARKETO_REST_SECRET = auth[:MARKETO_REST_SECRET]
+MARKETO_SUBDOMAIN   = auth[:MARKETO_SUBDOMAIN]
+MARKETO_API_LIMIT   = auth[:MARKETO_API_LIMIT]
+LEAD_LIST_DUMP_CSV  = auth[:LEAD_LIST_DUMP_CSV]
+GOODDATA_USER       = auth[:GOODDATA_USER]
+GOODDATA_PASSWORD   = auth[:GOODDATA_PASSWORD]
+GOODDATA_PROJECT    = auth[:GOODDATA_PROJECT]
+GOODDATA_ADS        = auth[:GOODDATA_ADS]
+
+S3_PUBLIC_KEY       = auth[:S3_PUBLIC_KEY]
+S3_PRIVATE_KEY      = auth[:S3_PRIVATE_KEY]
+S3_BUCKET           = auth[:S3_BUCKET]
+
+# No need to configure beyond this point. #
+@s3 = S3Helper.new :public_key => S3_PUBLIC_KEY,
+                   :private_key => S3_PRIVATE_KEY,
+                   :bucket => S3_BUCKET
+
+@webdav = WebDAV.new(:user => GOODDATA_USER,
+                     :pass => GOODDATA_PASSWORD,
+                     :project => GOODDATA_PROJECT)
+
+@dwh = GoodData::Datawarehouse.new(GOODDATA_USER,
+                                   GOODDATA_PASSWORD,
+                                   GOODDATA_ADS)
+
+@marketo = GoodDataMarketo.connect(:user_id => MARKETO_SOAP_USER,
+                                   :encryption_key => MARKETO_SOAP_KEY,
+                                   :api_subdomain => MARKETO_SUBDOMAIN,
+                                   :webdav => @webdav)
+
+GoodDataMarketo.logging = true
+
+# Test services
+# @marketo.test_rest
+# @marketo.test_soap
+# @s3.test
+# @dwh.test
+# @webdav.test
+
+def run_load config = {}
+
+  index = config[:index] || 1
+  @increment = config[:increment] || (12*60*60)
+  @marketo = config[:marketo_client]
+  @ads_target_table_name = config[:ads_table]
+  @lead_dump_file = "get_load_chunk_#{index}"
+
+  if @s3.exists? 'queue.json'
+
+    @queue = JSON.parse(@s3.download('queue.json'))
+    # Cancel the load if the queue is empty and delete the object.
+    if @queue.empty?
+      puts 'WARNING: Empty queue array was extracted from S3. Using queue passed in method.' if GoodDataMarketo.logging
+      @s3.delete('queue.json')
+      @queue = config[:queue]
+
+    end
+
+  else
+    @queue = config[:queue]
+  end
+
+  raise "You must pass an array of job hashs :queue_array when using run_load AND define a :marketo_client." unless @queue.length > 0 && @marketo
+  raise ":ads_table param is required with using run_load." unless @ads_target_table_name
+
+  loop do
+
+    loads = @marketo.loads(:user => GOODDATA_USER,
+                           :pass => GOODDATA_PASSWORD,
+                           :project => GOODDATA_PROJECT,
+                           :marketo_client => @marketo)
+
+    if loads.available?
+
+      file = loads.available.first
+
+      load = loads.create :name => file
+
+      @job_name = file
+      @id = load.id
+      @ads_table = load.json[:ads_table]
+      # Run the load from local or remote.
+      load.execute
+      # Data from the job can now be accessed ARRAY load.storage
+      # load.storage
+
+      if !load.storage.empty?
+
+        # Join all of the columns from the sample to all other columns.
+
+        @columns_load_aggregate = ['sys_capture_date']
+        load.storage.each { |raw_json_lead|
+
+          lead = GoodDataMarketo::Activity.new raw_json_lead if load.json[:method] == 'get_changes'
+          lead = GoodDataMarketo::Lead.new raw_json_lead if load.json[:method] == 'get_multiple'
+
+          print "\r#{Time.now} => Resolving columns: #{lead.id}\s" if GoodDataMarketo.logging
+
+          @columns_load_aggregate = @columns_load_aggregate | lead.columns
+        }
+        @columns_load_aggregate.map! { |column|
+          column.downcase.gsub('-','_')
+        }
+
+        # DEFAULTS: Use the correct ADS table.
+        if load.json[:method] == "get_multiple"
+          ads_target_table_name = 'marketo_leads'
+        elsif load.json[:method] = 'get_changes'
+          ads_target_table_name = 'marketo_changes'
+        else
+          ads_target_table_name = 'dump'
+        end
+
+        # Set up a new Table/Automatically loads current table if exists.
+
+        table = Table.new :client => @dwh, :name => @ads_table || ads_target_table_name, :columns => ['id','sys_capture_date']
+
+        if @columns_load_aggregate.length > 0
+          table.merge_columns :merge_with => @columns_load_aggregate
+
+        end
+        @csv = CSV.open("#{@id}.csv", 'w')
+
+        updated_columns = table.columns
+
+        if @columns_load_aggregate.length > 0
+          @csv << updated_columns
+        end
+
+        count = 0
+
+        ids_for_get_multiple_load = []
+
+        puts "#{Time.now} => Building objects from stream cache." if GoodDataMarketo.logging
+
+        total_leads = load.storage.length
+        total_leads_index = 0
+        load.storage.pmap do |lead|
+
+          lead = GoodDataMarketo::Activity.new lead if load.json[:method] == 'get_changes'
+          lead = GoodDataMarketo::Lead.new lead if load.json[:method] == 'get_multiple'
+
+          total_leads_index += 1
+          percentage = ((100*total_leads_index.to_f)/total_leads.to_f).round(1)
+          print "\r#{Time.now} => Transforming #{lead.id} #{percentage}%\s" if GoodDataMarketo.logging
+
+          # Get any new lead or merge lead ids and queue them for a load with get multiple.
+          ids_for_get_multiple_load << lead.values['merge_id'] if lead.values['merge_id']
+          ids_for_get_multiple_load << lead.values['lead_id'] if lead.values['lead_id']
+
+          row_to_save_csv = []
+
+          row_with_columns = updated_columns.map { |column|
+            if lead.columns.include? column
+              { column => lead.values[column] }
+            elsif lead.columns.include? "#{column}_m" # Check for anything that was removed by SQL
+              { "#{column}_m" => lead.values["#{column}_m"] }
+            elsif column == 'sys_capture_date'
+              { 'sys_capture_date' => Time.now.to_s }
+            else
+              { column => nil }
+            end
+          }
+
+          row_with_columns.each { |item|
+            c = item.to_a.flatten
+            if c[1] == nil
+              row_to_save_csv << nil
+            else
+              row_to_save_csv << c[1]
+            end
+
+          }
+
+          count += 1
+          @csv << row_to_save_csv
+
+        end
+
+        # Prepare (flush) the CSV for upload.
+        @csv.flush
+
+        table.import_csv("#{@id}.csv")
+
+        puts "#{Time.now} => #{@dwh.execute_select("SELECT id FROM #{@ads_table}").length} rows in ADS." if GoodDataMarketo.logging
+        puts "#{Time.now} => Rows extracted from CSV: #{count}" if GoodDataMarketo.logging
+        puts "#{Time.now} => New and merged ids queued: #{ids_for_get_multiple_load.length}" if GoodDataMarketo.logging
+
+        save_ids_for_get_multiple ids_for_get_multiple_load, 'a'
+
+        File.delete("#{load.json[:name]}_load.json") if File.exists? ("#{load.json[:name]}_load.json")
+        File.delete("#{@id}.csv") if File.exists? ("#{@id}.csv")
+
+        puts "#{Time.now} => Ads import of \"#{@lead_dump_file}\" complete." if GoodDataMarketo.logging
+        puts "#{Time.now} => Arguments: #{load.arguments}" if GoodDataMarketo.logging
+
+      end
+
+      case load.json[:method]
+
+        when 'get_changes'
+
+          # Increment the load by one day if it is time related.
+
+          oca = load.arguments[:oldest_created_at]
+          lca = load.arguments[:latest_created_at]
+
+          increment = Time.parse(lca) - Time.parse(oca)
+          total_time_range = Time.now - Time.parse(oca)
+
+          load.arguments[:oldest_created_at] = Time.parse(lca).to_s
+          load.arguments[:latest_created_at] = (Time.parse(lca) + increment).to_s
+
+          puts "#{Time.now} => API calls until current time: #{(total_time_range/increment).round}" if GoodDataMarketo.logging
+
+          # If the latest time is later then today kill the load.
+
+          if Time.parse(load.arguments[:latest_created_at]) > Time.now
+
+
+            load.terminate
+
+            next
+
+            # Otherwise save the load and resume additional loads.
+          else
+
+            load.save
+
+            next
+
+          end
+
+        when 'get_multiple'
+
+          determine_loads_state
+
+        else
+
+          raise 'Unable to determine lead type ("get_multiple"/"get_changes")!'
+
+          break
+
+      end
+
+    else
+
+      load = @queue.pop
+
+      puts "#{Time.now} => #{@queue.length} loads remaining."
+
+      if @queue.length > 0
+        File.open('queue.json','w'){ |f| JSON.dump(@queue, f) }
+        @s3.upload('queue.json')
+      end
+
+      if load
+
+        loads.create load
+
+        next
+
+      else
+
+        @s3.delete('queue.json')
+        File.delete('queue.json') if File.exists?('queue.json')
+
+        break
+
+      end
+
+
+    end
+
+
+  end
+
+end
+
+#####################################
+#                                   #
+#  INITIAL GET ALL MULTIPLE LEADS   #
+#                                   #
+#####################################
+# Downloads all current lead ids with REST ID.
+# Uses SOAP API to rotate through ids with get multiple.
+# Runs once, boolean value set in initial_load_get_multiple at marketo_connector_config.json
+
+def initial_load_get_multiple
+
+  @marketo.write_all_lead_ids_to_csv # Large bulk download to CSV of leads over REST API.
+
+  ids = CSV.open(LEAD_LIST_DUMP_CSV).map { |m| m[0] }
+
+  puts "#{Time.now} => #{ids.length} imported from local CSV." if GoodDataMarketo.logging
+
+  counter = 0
+
+  loop do
+
+    counter += 1
+
+    batch = ids.slice!(1..1000)
+
+    break if batch.length <= 0
+
+    get_multiple_leads_configuration = {
+        :name => "get_all_leads_chunk",
+        :type => 'leads',
+        :method => 'get_multiple',
+        :ads_table => 'marketo_leads',
+        :arguments => {
+            :ids => batch, # Notice the addition of the IDS box
+            :type => 'IDNUM'
+        }
+    }
+
+    puts "#{Time.now} => Id count in batch:#{batch.length} (Req:#{counter})" if GoodDataMarketo.logging
+
+    run_load :batch => batch,
+             :counter => counter,
+             :ads_table => 'marketo_leads',
+             :marketo_client => @marketo,
+             :queue => [get_multiple_leads_configuration]
+
+    save_ids_for_get_multiple ids, 'w'
+
+  end
+end
+
+########################################
+#                                      #
+#     INITIAL GET ALL LEAD CHANGES     #
+#                                      #
+########################################
+# Starts January 1st 2000 and increments by the day until present day.
+# All Activity Types included unless specified.
+# Runs once, boolean value set in initial_load_get_changes at marketo_connector_config.json
+
+def initial_load_get_changes
+
+  get_lead_changes_configuration = {
+      :name => 'get_lead_changes_chunk',
+      :type => 'leads',
+      :method => 'get_changes',
+      :ads_table => 'marketo_changes',
+      :arguments => {
+          # "oldest_created_at" and "latest_created_at" is the size of the increment it will to current time.
+          :oldest_created_at => 'January 1st 2004',
+          :latest_created_at => 'January 2nd 2004',
+          :filters => []
+      }
+  }
+
+  # Create a new configuration object for each activity type.
+  get_lead_changes_configuration.freeze
+
+  queue = []
+
+  @marketo.activity_types.each { |type|
+
+    g = {
+        :filters => [type]
+    }
+
+    m = get_lead_changes_configuration.dup
+    c = m.dup
+    c[:arguments] = m[:arguments].merge(g)
+
+    queue << c
+
+  }
+
+  run_load :ads_table => get_lead_changes_configuration[:ads_table],
+           :queue => queue,
+           :marketo_client => @marketo,
+           :index => 1,
+           :counter => 1
+
+  puts "#{Time.now} => Updating initial load changes to true in connector configuration." if GoodDataMarketo.logging
+
+  @s3.set_config(:initial_load_get_changes => true)
+
+end
+
+########################################
+#                                      #
+#        SYNC MULTIPLE LEADS           #
+#                                      #
+########################################
+
+def update_get_multiple_leads
+
+  file = File.open(LEAD_LIST_DUMP_CSV, 'w')
+  file.puts @s3.download(LEAD_LIST_DUMP_CSV)
+
+  ids = CSV.open(LEAD_LIST_DUMP_CSV).map { |m| m[0] }
+
+  puts "#{Time.now} => #{ids.length} imported from local CSV." if GoodDataMarketo.logging
+
+  counter = 0
+
+  loop do
+
+    counter += 1
+
+    batch = ids.slice!(1..1000)
+
+    break if batch.length <= 0
+
+    get_multiple_leads_configuration = {
+        :name => "get_all_leads_chunk",
+        :type => 'leads',
+        :method => 'get_multiple',
+        :ads_table => 'marketo_leads',
+        :arguments => {
+            :ids => batch, # Notice the addition of the IDS box
+            :type => 'IDNUM'
+        }
+    }
+
+    puts "#{Time.now} => Id count in batch:#{batch.length} (Req:#{counter})" if GoodDataMarketo.logging
+
+    run_load :batch => batch,
+             :counter => counter,
+             :ads_table => 'marketo_leads',
+             :marketo_client => @marketo,
+             :queue => [get_multiple_leads_configuration]
+
+    save_ids_for_get_multiple ids, 'w'
+
+  end
+end
+
+########################################
+#                                      #
+#          SYNC LEAD CHANGES           #
+#                                      #
+########################################
+# Synchronizes last 12 hours from process execution.
+# Runs continuously.
+
+def update_lead_changes
+
+  # Changing activity types from the default set to just Visit Webpage type.
+  #@marketo.activity_types = ['Visit Webpage']
+  #@marketo.activity_types = ['New SFDC Opportunity','Remove from Opportunity','Add to Opportunity', 'Update Opportunity']
+
+  fourty_eight_hours_ago = (Time.now - (48*60*60)).to_s
+  twenty_four_hours_ago = (Time.now - (24*60*60)).to_s
+  twelve_hours_ago = (Time.now - (12*60*60)).to_s
+  six_hours_ago = (Time.now - (6*60*60)).to_s
+
+  get_lead_changes_configuration = {
+      :name => 'get_lead_changes_chunk',
+      :type => 'leads',
+      :method => 'get_changes',
+      :ads_table => 'marketo_changes_date_platform',
+      :arguments => {
+          # "oldest_created_at" and "latest_created_at" is the size of the increment it will to current time.
+          :oldest_created_at => twelve_hours_ago,
+          :latest_created_at => six_hours_ago,
+          :filters => []
+      }
+  }
+
+  # Create a new configuration object for each activity type.
+  get_lead_changes_configuration.freeze
+
+  queue = []
+
+  @marketo.activity_types.each { |type|
+
+    g = {
+        :filters => [type]
+    }
+
+    m = get_lead_changes_configuration.dup
+    c = m.dup
+    c[:arguments] = m[:arguments].merge(g)
+
+    queue << c
+
+  }
+
+  run_load :ads_table => get_lead_changes_configuration[:ads_table],
+           :queue => queue,
+           :marketo_client => @marketo,
+           :increment => (24*60*60),
+           :counter => 1
+
+end
+
+def save_ids_for_get_multiple ids, write_type
+  csv = CSV.open(LEAD_LIST_DUMP_CSV, write_type)
+  ids.each { |row| csv << [row] }
+  csv.flush
+  @s3.upload(LEAD_LIST_DUMP_CSV)
+end
+
+# Download the configuration file from S3. If there is not one, one will be created.
+config = @s3.get_config
+
+if !config[:initial_load_get_multiple]
+
+  initial_load_get_multiple
+
+# elsif !config[:initial_load_get_changes]
+#
+#   initial_load_get_changes
+
+else
+
+  update_lead_changes
+  #
+  # #Once the most recent changes downloaded, create and save a load for update_get_multiple_leads
+  # update_get_multiple_leads
+
+end