gooddata_eloqua 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 833a93676772e7e93b3745dd7f9e7ba9a9633f91
+  data.tar.gz: 6e1e320b6553360d9a1d0f14606cbe36d90b2284
+SHA512:
+  metadata.gz: 4847d22bd39cbfbf285039a989753a769d9c70fbf346bbd469a8eead823405fe1b2b5febb1044d0404711f91ec8f9eee611a2833a6c6ba3aa00977fe41b592b2
+  data.tar.gz: 0fa933a67e09c8353b9ae74592087e61bbad434f4b29ddc01f891ed3a9102e01b7fe4f629fddc2f3681f875f362571ca8c6bb5983665f4b79d91a6d9f2e54d50

data/lib/gooddata_eloqua.rb

@@ -0,0 +1,12 @@
+
+module GoodDataEloqua
+  class << self
+    def connect eloqua_interface
+      GoodDataEloqua::Client.new(:eloqua => eloqua_interface)
+    end
+
+    alias :client :connect
+  end
+end
+
+require_relative 'gooddata_eloqua/client'

data/lib/gooddata_eloqua/client.rb

@@ -0,0 +1,49 @@
+require 'csv'
+require 'securerandom'
+require 'fileutils'
+require 'pmap'
+require 'eloqua_api'
+require 'pry'
+require 'base64'
+require 'rest_client'
+
+class GoodDataEloqua::Client
+
+  def initialize(config={})
+    $global_id = SecureRandom.hex
+    $client = config[:eloqua] || config[:client]
+
+    self.system_check
+
+  end
+
+  def campaigns(config={})
+    GoodDataEloqua::Campaigns.new(config)
+  end
+
+  def emails(config={})
+    GoodDataEloqua::Emails.new(config)
+  end
+
+  def contacts(config={})
+    GoodDataEloqua::Contacts.new(config)
+  end
+
+  def system_check(refresh = false)
+
+    if refresh
+      FileUtils.rm_rf('downloads')
+    end
+
+    FileUtils.mkdir_p 'downloads'
+
+  end
+
+end
+
+require_relative 'helpers/pool'
+require_relative 'helpers/request'
+
+require_relative 'models/campaigns'
+require_relative 'models/emails'
+require_relative 'models/contacts'

data/lib/gooddata_eloqua/helpers/pool.rb

@@ -0,0 +1,32 @@
+class Pool
+
+  attr_accessor :running
+
+  def initialize
+    @group = ThreadGroup.new
+    @running = true
+  end
+
+  def thread(&thread)
+    @group.add(Thread.new(&thread))
+  end
+
+  alias :fork :thread
+
+  def active_threads
+    @group.list.length
+  end
+
+  def list
+    @group.list
+  end
+
+  def exit
+
+    @group.pmap { |thread|
+      thread.exit
+    } unless @group.empty?
+
+  end
+
+end

data/lib/gooddata_eloqua/helpers/request.rb

@@ -0,0 +1,264 @@
+class GoodDataEloqua::Request
+
+  attr_accessor :asset_type
+
+  def initialize(config = {})
+
+    @client = $client
+    @asset_type_plural = nil
+    @asset_type_singular = nil
+
+  end
+
+  def get_all(config = {})
+    end_point = @asset_type_plural || config[:end_point] || config[:url] || @parent
+    page = config[:page] || config[:page_number]
+
+    raise ':end_point and :page_number must be defined.' unless end_point && page
+
+    @client.get(end_point, "?page=#{page}")
+
+  end
+
+  def get_all_by_page(page, config ={})
+
+    default = Hash.new
+    default[:end_point] = @asset_type_plural || config[:end_point]
+    default[:page] = page
+    self.get_all(default)
+
+  end
+
+  def write_all_ids_return_csv(file_name, config={})
+
+    if file_name
+      fd = IO.sysopen("./downloads/#{file_name}", 'w+')
+    end
+
+    csv = CSV.new(IO.new(fd))
+    analyze = self.get_all_by_page(0)
+    pages = analyze['total'].to_i
+    total = (pages/1000).round
+
+    total = 1 if total == 0
+
+    self.get_all_distributed total, csv
+    puts "\n#{Time.now} => Flushing IO (#{@session_csv}.id)..."
+
+
+    file_name
+
+  end
+
+  def ids_to_eloqua_return_csv(csv, config = {})
+
+    if csv
+      fd = IO.sysopen("./downloads/#{csv}", 'r')
+    end
+
+    started_method = Time.now
+
+    write_path = "./downloads/#{@session_id}_complete.csv"
+    write_fd = IO.sysopen(write_path, 'w+')
+    csv = CSV.new(IO.new(write_fd))
+    pool = Pool.new
+
+    CSV.open(fd) do |read_csv|
+
+      ids = read_csv.map { |row|
+        row[0]
+      }
+
+      puts "#{Time.now} => Extracted from CSV: #{ids.length}\n"
+
+      elapsed_times = []
+
+      loop do
+
+        batch = ids.slice!(1..20)
+
+        started_batch = Time.now
+
+        if elapsed_times.length > 5
+          estimated_time = elapsed_times.inject{ |sum, el| sum + el }.to_f / elapsed_times.size
+        else
+          estimated_time = 0
+        end
+
+        if (estimated_time/60) > 60 && estimated_time != 0
+          time_readable = "#{((estimated_time/60)/60).round(2)} hours"
+
+        elsif (estimated_time/60) < 60 && (estimated_time/60) > 1 && estimated_time != 0
+          time_readable = "#{(estimated_time/60).round(2)} minutes"
+
+        elsif (estimated_time/60) > 0 && estimated_time != 0
+          time_readable = "#{(estimated_time).round(2)} seconds"
+
+        else
+          time_readable = 'Estimating...'
+
+        end
+
+        break unless batch.length > 0 && pool.list.length
+
+        #
+        batch.each { |i|
+          pool.thread {
+            response = self.get_one_by_id(i)
+            csv << response.values
+            csv.flush
+            print "\r#{Time.now} => IDs Remaing: #{ids.length} - Complete: #{time_readable} - Thread Pool: #{pool.active_threads} \s\s\s\s"
+          }
+        }
+
+        ended_batch = Time.now
+
+        elapsed_time = ended_batch - started_batch
+        number_of_estimated_chunks = ids.length/20
+        estimated_batch_time = (elapsed_time*number_of_estimated_chunks).round
+
+        if elapsed_times.length < 10
+          elapsed_times << estimated_batch_time
+        elsif elapsed_times.length > 50
+          40.times do
+            elapsed_times.pop
+          end
+        end
+
+
+        if pool.active_threads > 21
+          sleep 3
+        end
+
+        break if ids.length == 0 && pool.list.length
+
+
+      end
+
+
+
+      puts "#{Time.now} => Complete. CSV written to \"./downloads/#{@session_id}_complete.csv\""
+
+    end
+
+    write_path
+
+  end
+
+
+  def get_all_distributed num_iterations, csv
+
+    iterations = []
+    num_iterations.times { |i| iterations << i }
+    iterations << num_iterations
+
+    pool = Pool.new
+
+    while pool.running do
+
+      batch = iterations.slice!(1..20)
+
+      break unless batch
+
+      batch.each { |i|
+        pool.thread {
+          response = self.get_all_by_page(i)
+          response['elements'].each { |element|
+            csv << [element['id']]
+          }
+          csv.flush
+          print "\r#{Time.now} => Pages #{iterations.length} of #{num_iterations} - Workers: #{pool.active_threads}\s"
+        }
+      }
+
+      if pool.active_threads > 20
+        sleep 2
+      end
+
+      if iterations.length == 0 && pool.active_threads == 0
+        pool.running = false
+      else
+        next
+      end
+
+    end
+
+  end
+
+  def set_asset_type asset
+
+    case asset
+      when 'contact', 'contacts'
+
+        if asset[-1] == 's'
+          @asset_type_plural = "/data/#{asset}"
+          @asset_type_singular = "/data/#{asset[0..-2]}"
+        elsif asset[-1] != 's'
+          @asset_type_plural = "/data/#{asset}s"
+          @asset_type_singular = "/data/#{asset}"
+        else
+          raise "Unable to parse \"#{asset}\" please make sure it is a string."
+        end
+
+      when 'email', 'emails'
+
+        if asset[-1] == 's'
+          @asset_type_plural = "/assets/#{asset}"
+          @asset_type_singular = "/assets/#{asset[0..-2]}"
+        elsif asset[-1] != 's'
+          @asset_type_plural = "/assets/#{asset}s"
+          @asset_type_singular = "/assets/#{asset}"
+        else
+          raise "Unable to parse \"#{asset}\" please make sure it is a string."
+        end
+
+      when 'campaign', 'campaigns'
+
+        if asset[-1] == 's'
+          @asset_type_plural = "/assets/#{asset}"
+          @asset_type_singular = "/assets/#{asset[0..-2]}"
+        elsif asset[-1] != 's'
+          @asset_type_plural = "/assets/#{asset}s"
+          @asset_type_singular = "/assets/#{asset}"
+        else
+          raise "Unable to parse \"#{asset}\" please make sure it is a string."
+        end
+
+      else
+
+        if asset[-1] == 's'
+          @asset_type_plural = "/assets/#{asset}"
+          @asset_type_singular = "/assets/#{asset[0..-2]}"
+        elsif asset[-1] != 's'
+          @asset_type_plural = "/assets/#{asset}s"
+          @asset_type_singular = "/assets/#{asset}"
+        else
+          raise "Unable to parse \"#{asset}\" please make sure it is a string."
+        end
+
+    end
+
+  end
+
+  alias :get_all_from_page :get_all_by_page
+
+  def get_one(config={})
+    #lookup_key = @asset_type_singular.gsub("/assets/","").downcase.to_sym
+    sym_key = config[:id]
+    depth = config[:depth] || 'complete'
+    end_point = @asset_type_singular+"/"+sym_key
+    @client.get(end_point, depth)
+  end
+
+  def get_one_by_id(id, config={})
+    default = Hash.new
+    default[:depth] = config[:depth] || 'complete'
+    default[:id] = id
+    self.get_one(default)
+  end
+
+  def clean
+    FileUtils.rm_f("/downloads/#{@session_csv}")
+  end
+
+end

data/lib/gooddata_eloqua/models/campaigns.rb

@@ -0,0 +1,15 @@
+
+class GoodDataEloqua::Campaigns < GoodDataEloqua::Request
+
+  def initialize(config={})
+    @client = $client
+
+    @client = $client
+    @session_id = SecureRandom.hex+"_campaigns"
+    @session_csv = @session_id+".csv"
+    self.set_asset_type('campaigns')
+
+  end
+
+
+end

data/lib/gooddata_eloqua/models/contacts.rb

@@ -0,0 +1,15 @@
+class GoodDataEloqua::Contacts < GoodDataEloqua::Request
+
+  attr_accessor :session_csv
+
+  def initialize(config={})
+
+    @client = $client
+    @session_id ||= SecureRandom.hex+"_contacts"
+    @session_csv = @session_id+".csv"
+    self.set_asset_type('contacts')
+
+    #""/data/contact/100002'
+  end
+
+end

data/lib/gooddata_eloqua/models/emails.rb

@@ -0,0 +1,15 @@
+
+class GoodDataEloqua::Emails < GoodDataEloqua::Request
+
+  attr_accessor :session_csv
+
+  def initialize(config={})
+
+    @client = $client
+    @session_id ||= SecureRandom.hex+"_emails"
+    @session_csv = @session_id+".csv"
+    self.set_asset_type('emails')
+
+  end
+
+end

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification
+name: gooddata_eloqua
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Patrick McConlogue
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-02-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: eloqua_api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.11
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.61.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.61.0
+- !ruby/object:Gem::Dependency
+  name: pmap
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+- !ruby/object:Gem::Dependency
+  name: gooddata
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.12
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.12
+description: Eloqua REST API to CSV
+email: patrick.mcconlogue@gooddata.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/gooddata_eloqua.rb
+- lib/gooddata_eloqua/client.rb
+- lib/gooddata_eloqua/helpers/pool.rb
+- lib/gooddata_eloqua/helpers/request.rb
+- lib/gooddata_eloqua/models/campaigns.rb
+- lib/gooddata_eloqua/models/contacts.rb
+- lib/gooddata_eloqua/models/emails.rb
+homepage: https://github.com/thnkr/connectors/tree/master/eloqua
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Eloqua to CSV
+test_files: []
+has_rdoc: