gooddata_marketo 0.0.1-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 522a33b780f99c3d01e5bec4e9e26708b754c5ab
+  data.tar.gz: adb50eb798055b9d6adcb74e9b5d1b76c2de0fde
+SHA512:
+  metadata.gz: 431cd59dae68255b34126f7a047dbe81cad44620fc5059de14a5c4918b8be685ee5638a1406d75cd75c6f584c376ccf5d705cbb4f9b3805fb2347da0e76e2f1f
+  data.tar.gz: 21eb2af9c8ea6eb8818486885b4ce979dfc459ea554bc5ddab24d5ecf81be55d2d108af8b9519b0cdb4faeac77b7e4003f4c623272af66d17ead7a3859fec442

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+gem 'savon', '2.8.0'
+gem 'rubyntlm', '0.3.2'
+gem 'gooddata', '0.6.11'
+gem 'rest-client', '1.7.2'
+gem 'aws-sdk', '1.61.0'
+gem 'pmap', '1.0.2'
+gem 'gooddata_warehouse', '0.0.5'

data/Gemfile.lock

@@ -0,0 +1,131 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.0)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    akami (1.2.2)
+      gyoku (>= 0.4.0)
+      nokogiri
+    aws-sdk (1.61.0)
+      aws-sdk-v1 (= 1.61.0)
+    aws-sdk-v1 (1.61.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+    builder (3.2.2)
+    coderay (1.1.0)
+    docile (1.1.5)
+    erubis (2.7.0)
+    faraday (0.9.1)
+      multipart-post (>= 1.2, < 3)
+    faraday_middleware (0.9.1)
+      faraday (>= 0.7.4, < 0.10)
+    ffi (1.9.6-java)
+    gli (2.12.2)
+    gooddata (0.6.11)
+      activesupport (~> 4.1, >= 4.1.0)
+      aws-sdk (~> 1.45)
+      bundler (~> 1.7, >= 1.7.3)
+      docile (~> 1.1, >= 1.1.5)
+      erubis (~> 2.7, >= 2.7.0)
+      gli (~> 2.12, >= 2.12.2)
+      highline (~> 1.6, >= 1.6.21)
+      i18n (~> 0.6, >= 0.6.9)
+      json_pure (~> 1.8, >= 1.8.1)
+      multi_json (~> 1.10, >= 1.10.0)
+      parseconfig (~> 1.0, >= 1.0.4)
+      pmap (~> 1.0, >= 1.0.1)
+      pry (~> 0.9.12.6)
+      rest-client (~> 1.7, >= 1.7.2)
+      restforce (~> 1.5, >= 1.5.0)
+      rubyzip (~> 1.1, >= 1.1.0)
+      salesforce_bulk_query (~> 0.0)
+      terminal-table (~> 1.4, >= 1.4.5)
+    gyoku (1.2.2)
+      builder (>= 2.1.2)
+    hashie (3.3.2)
+    highline (1.6.21)
+    httpi (2.3.0)
+      rack
+    i18n (0.7.0)
+    json (1.8.2)
+    json (1.8.2-java)
+    json_pure (1.8.2)
+    macaddr (1.7.1)
+      systemu (~> 2.6.2)
+    method_source (0.8.2)
+    mime-types (1.25.1)
+    mini_portile (0.6.2)
+    minitest (5.5.0)
+    multi_json (1.10.1)
+    multipart-post (2.0.0)
+    netrc (0.10.2)
+    nokogiri (1.6.5)
+      mini_portile (~> 0.6.0)
+    nokogiri (1.6.5-java)
+    nori (2.4.0)
+    parseconfig (1.0.6)
+    pmap (1.0.2)
+    pry (0.9.12.6)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    pry (0.9.12.6-java)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+      spoon (~> 0.0)
+    rack (1.6.0)
+    rest-client (1.7.2)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    restforce (1.5.1)
+      faraday (~> 0.9.0)
+      faraday_middleware (>= 0.8.8)
+      hashie (>= 1.2.0, < 4.0)
+      json (>= 1.7.5, < 1.9.0)
+    rubyntlm (0.3.2)
+    rubyzip (1.1.6)
+    salesforce_bulk_query (0.0.6)
+      json (~> 1.8)
+      xml-simple (~> 1.1)
+    savon (2.8.0)
+      akami (~> 1.2)
+      builder (>= 2.1.2)
+      gyoku (~> 1.2)
+      httpi (~> 2.3)
+      nokogiri (>= 1.4.0)
+      nori (~> 2.4)
+      uuid (~> 2.3.7)
+      wasabi (= 3.3.0)
+    slop (3.6.0)
+    spoon (0.0.4)
+      ffi
+    systemu (2.6.4)
+    terminal-table (1.4.5)
+    thread_safe (0.3.4)
+    thread_safe (0.3.4-java)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    uuid (2.3.7)
+      macaddr (~> 1.0)
+    wasabi (3.3.0)
+      httpi (~> 2.0)
+      mime-types (< 2.0.0)
+      nokogiri (>= 1.4.0)
+    xml-simple (1.1.4)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  aws-sdk (= 1.61.0)
+  gooddata (= 0.6.11)
+  pmap (= 1.0.2)
+  rest-client (= 1.7.2)
+  rubyntlm (= 0.3.2)
+  savon (= 2.8.0)

data/README.md

@@ -0,0 +1,207 @@
+GOODDATA MARKETO CONNECTOR
+==========================
+Marketo SOAP/REST services to GoodData ADS
+
+## Overview
+With the Marketo Connector a complete transfer of all available lead data is automatically moved *and* kept in sync with GoodData ADS. Supporting both the Marketo REST & SOAP the GoodData Marketo Connector adapts to the source data making installation as simple as entering your login information.
+
+### Marketo
+The central concept of the Marketo API is a "Lead". Each lead contains default attributes like City or Email in addition to custom attributes added by the client. The Marketo Connector downloads and synchronizes all leads and lead changes for a given Marketo client. The connector utitlizes two main calls, *get_lead_changes* and *get_multiple_leads*, these feed into the ADS tables *marketo_changes* and *marketo_leads* respectively. For examples, take a look at the code within the *examples* directory.
+
+### Getting Started
+
+1. Install jruby `rvm install jruby` and then `rvm use jruby` (tested on jruby 1.7.16.1)
+2. [Install DSS for Ruby](https://confluence.intgdc.com/pages/viewpage.action?title=DSS+Alpha+Version+-+User+Guide&spaceKey=plat)
+3. Clone this repository.
+4. Configure authentication in bin/auth.json.
+5. Zip the the entire "bin" folder to a new zip file.
+6. Upload the zip file it as a Ruby process, schedule it to run daily, every 12 hours.
+
+### Configuration & Keys
+To use the Marketo Connector you must have credentials for the following services.
+
+- [Marketo REST API](http://developers.marketo.com/blog/quick-start-guide-for-marketo-rest-api/).
+- [Marketo SOAP API](http://developers.marketo.com/documentation/soap/).
+- [GoodData Username, Password, & Project ID](https://na1.secure.gooddata.com/account.html?clicked_trial_link=1#/registration/projectTemplate/urn%3Agooddata%3AOnboarding).
+- [AWS S3](http://aws.amazon.com/s3/).
+
+#### File Tree
+
+    ├── bin
+    │   ├── Gemfile
+    │   ├── auth.json
+    │   ├── main.rb
+    │   └── process.rbx
+    │
+    ├── examples
+    │   ├── all_lead_changes.rb
+    │   ├── all_leads.rb
+    │   └── lead_changes_to_ads.rb
+    │
+    ├── lib
+    │   ├── gooddata_marketo
+    │   │   ├── adapters
+    │   │   │   └── rest.rb
+    │   │   ├── client.rb
+    │   │   ├── data
+    │   │   │   ├── activity_types.rb
+    │   │   │   └── reserved_sql_keywords.rb
+    │   │   ├── helpers
+    │   │   │   ├── s3.rb
+    │   │   │   ├── stringwizard.rb
+    │   │   │   ├── table.rb
+    │   │   │   └── webdav.rb
+    │   │   ├── loads.rb
+    │   │   └── models
+    │   │       ├── campaigns.rb
+    │   │       ├── channels.rb
+    │   │       ├── child
+    │   │       │   ├── activity.rb
+    │   │       │   ├── criteria.rb
+    │   │       │   ├── lead.rb
+    │   │       │   └── mobj.rb
+    │   │       ├── etl.rb
+    │   │       ├── leads.rb
+    │   │       ├── load.rb
+    │   │       ├── mobjects.rb
+    │   │       ├── streams.rb
+    │   │       ├── tags.rb
+    │   │       └── validate.rb
+    │   └── gooddata_marketo.rb
+    │
+    └── tests
+        └── test.rb
+
+## Custom Scripts
+In addition to running the main process you can also write scripts for exploration and small transformations with the Marketo API. To start, require the Marketo Connector and create a client.
+
+    require 'marketo_connector'
+
+    user = ""
+    key = ""
+    subdomain = ""
+
+    client = GoodDataMarketo.connect(:user_id => user,
+                                      :encryption_key => key,
+                                      :api_subdomain => subdomain)
+
+From the client you can get a specific lead or a stream of ids.
+
+    lead = client.leads.get_by_email('email@email.com')
+    leads = client.leads.get_multiple ['23590','2930','9209'], :filters => 'Merge Leads', :type => 'IDNUM'
+
+## Customizing Calls
+Marketo API queries can be very time consuming, in the hours or days range, before a response returns. This is why the Marketo API allows for very complex filtration options like creation date, data-type, request size etc. Filtering your queries by some level is **required** due to the size and slower speed of Marketo. Keep in mind, as of this post RUBY Processes on the GoodData platform automatically terminate after 5 hours. Setting up configuration options for the two main calls, `client.leads.get_changes` and `client.leads.get_multiple` is almost identical, you can learn more about them [here](http://developers.marketo.com/documentation/soap/getmultipleleads/).
+
+#### Important Configurations
+
+* ` :timeout => Int `
+
+Specifies the amount of time before you want a call to the API to fail. Currently the library wide default is 120 which is also the the time allocated to the SAVON GEM in the initialization of the client. To go past that you will need to change the timeout when the client is first initialized.
+
+* ` :filters => Array `
+
+A list of activity types which are translated into the *activity_name_filters*. Due to changes in the Marketo API becareful to print these in complete form like 'Merge Leads' instead of 'MergeLeads' or 'Webpage Clicks' instead of 'webpageClicks'.
+
+* ` :type => String `
+
+Defines the type of your query. For instance, `:type => 'IDNUM'` would mean you are searching for data using a Marketo ID, or you might use `:type => 'EMAIL'` if you are querying using an email. In specific cases like `client.leads.get_by_id` the type is set for you.
+
+If the given call you are looking for is not fully implemented, you can always build the complete soap request your self by calling `client.call` directly.
+Outside of the authentication options, the default configuration for the client is in *marketo_connector/client.rb* under the parameter *DEFAULT_CONFIG*.
+
+## Leads
+All calls to Marketo return objects that contain attributes. Currently the Marketo Connector supports *Lead* and *Activity* where Activity is a child of lead containing the ID and values pertaining to the lead's activity. Custom attributes can be added to leads which include values, accessing them looks like this:
+
+    leads = client.leads.get_changes :last_created_at => '12/02/2013'
+    lead = leads.first
+    custom_form = lead.attributes['c__Custom_Form']
+
+Where `custom_form` will now contain the given value of the lead. Lead and Activity objects can also be flattened to save to CSV for parsing in the future.
+
+    lead = client.leads.get_by_email('john@smith.com')
+
+    CSV.open('leads.csv', 'wb') do |csv|
+
+      csv << lead.headers # => Incase you wanted to wanted the headers of attributes also saved to CSV.
+      csv << lead.to_row
+
+    end
+
+You can also extract the activities of a given lead through `lead.activities` or changes of a given lead with `lead.changes`. For example if you wanted to write the attributes for leads which included the City 'San Francisco' to csv...
+
+    leads = client.leads.get_changes :filters => ['Email Opens', 'Merge Leads'], :lastest_created_at => '10/11/2014'
+
+    san_francisco_leads = leads.select {|lead| lead.attributes['City'] == 'San Francisco'}
+
+    CSV.open('leads.csv', 'wb') do |csv|
+      san_francisco_leads.each do |lead|
+        csv << lead.to_row
+      end
+    end
+
+In addition, you can always work with the raw response of the lead object with `lead.raw`.
+
+## Streams
+Specific to getMultipleLeads, getLeadChanges, getMObjects, and getLeadActivity calls, the Marketo SOAP API supports "Streams" which allow you to increment through large responses by offset stamp or stream ID (just like pagenate).
+Within the client script (*lib/marketo_connector/client.rb*) are two calls, a streaming function which automatically works through APIs, building a complete object.
+
+In the event the Marketo times out **AND** you gave specific date configurations like *:started_at => TIME* or *:oldest_created_at => TIME*, the stream will increment your calls into smaller time windows until it does receive a response within the given timeout configuration. This is helpful to insure that the most relevant data is always in a clients account. You can also deactivate this by passing the configuration symbol *:safe* and setting it to **false**.
+
+## Custom Objects
+Marketo allows you to query the API using what it calls *criteria* which is a hash comprised of the attribute name and value with a comparison statement like *EQ* "Equals" or *LT* "Less Then". Many criteria hashes can be sent with a given request, due to the size of these queries it is also suggest that you do so but rather than forcing you to do with large arrays of criteria hashes you simply add criteria to the given *client.mobject* like so:
+
+    criteria_one = {
+        :attr_name => "Id", # See the types of content it can search above.
+        :comparison => "LE",
+        :attr_value => "1010"
+    }
+
+    >> client.mobjects.add(criteria_one)
+    [{:attr_name => 'Id', :comparison => 'LE', :attr_value => '1010'}]
+
+    criteria_two = {
+        :attr_name => "Zip", # See the types of content it can search above.
+        :comparison => "EQ",
+        :attr_value => "94104"
+    }
+
+    >> client.mobjects.add(criteria_two)
+    [{:attr_name => 'Id', :comparison => 'LE', :attr_value => '1010'}, {:attr_name => 'Zip', :comparison => 'EQ', :attr_value => '94104'}]
+
+The data is stored in a class array which allows you to do this:
+
+    objects = client.mobjects.get
+
+You can also make requests with one criteria hash like this:
+
+    client.mobjects.get :criteria => HASH
+
+Finally criteria can be removed by name with: ` client.mbobjects.remove_criteria 'NAME' `.
+
+## Supported Calls
+
+* [` client.leads.get_changes `](http://developers.marketo.com/documentation/soap/getleadchanges/) - Optional config. Streaming is required.
+
+* [` client.leads.get_multiple `](http://developers.marketo.com/documentation/soap/getmultipleleads/) - Requires an query Array, optional Config.
+
+* [` client.leads.get_lead `](http://developers.marketo.com/documentation/soap/getlead/) - Requires query String, optional config.
+
+* [` client.mobjects `](http://developers.marketo.com/documentation/soap/getcustomobjects/) - Requires criteria to be set (See Custom Objects above)
+
+* [` client.usage `](http://developers.marketo.com/documentation/rest/get-daily-usage/) -  Requires API key to within REST config.
+
+* [` client.channels `](http://developers.marketo.com/documentation/soap/getchannels/) - Requires query value, optional config.
+
+* [` client.tags `](http://developers.marketo.com/documentation/soap/gettags/) - Requires the Lead email or ID for the query, optional config.
+
+## Custom Requests
+You can call custom requests should you need want more specific control or use of a feature.
+
+#### Direct Call
+A single call is ` client.call(WEB_METHOD, MESSAGE) ` where WEB_METHOD is the specific SOAP function you are attempting to use and message is the SOAP request message (as a hash).
+
+#### Streaming Calls
+In addition for web methods that support streaming, you can using `client.stream(WEB_METHOD, MESSAGE)` to automatically iterate through either *offset* or *start_position* and return an array of results in the correct type.
+
+

data/bin/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+
+gem 'savon', '2.8.0'
+gem 'rubyntlm', '0.3.2'
+gem 'gooddata', '0.6.11'
+gem 'rest-client', '1.7.2'
+gem 'aws-sdk', '1.61.0'
+gem 'pmap', '1.0.2'
+gem 'gooddata_warehouse', '0.0.5'
+gem 'gooddata_marketo', '0.0.1'

data/bin/auth.json

@@ -0,0 +1,16 @@
+{
+	"MARKETO_SOAP_USER" : "",
+	"MARKETO_SOAP_KEY" : "",
+	"MARKETO_REST_ID" : "",
+  	"MARKETO_REST_SECRET" : "",
+    "MARKETO_SUBDOMAIN" : "123-456-789",
+	"MARKETO_API_LIMIT" : 10000,
+	"LEAD_LIST_DUMP_CSV" : "marketo_lead_dump.csv",
+	"GOODDATA_USER" : "you@email.com",
+	"GOODDATA_PASSWORD" : "",
+	"GOODDATA_PROJECT" : "wj82342uljb34dmz0mp92dsypuksv2j8xw5m",
+	"GOODDATA_ADS" : "t4d9806bbea8656285664cf0160d2342327",
+	"S3_PUBLIC_KEY" : "",
+	"S3_PRIVATE_KEY" : "",
+	"S3_BUCKET" : ""
+}

data/bin/main.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/ruby
+# !!! This file is an executable, do not edit. !!!
+
+RUBY_EXECUTABLE = 'process.rbx'
+
+files = [RUBY_EXECUTABLE, 'Gemfile'].map { |f| File.exists? f }
+
+raise 'ERROR: Unable to load necessary Gemfile and executable script!' unless files.all?
+
+# Installs dependencies for the process from Gemfile
+begin
+  system('bundle install')
+rescue Exception => exp
+  puts 'Broke executing bundle install'
+  puts exp
+
+end
+
+# Executes ruby process with bundler.
+begin
+  system("bundle exec ruby #{RUBY_EXECUTABLE}")
+rescue Exception => exp
+  puts exp
+end