gooddata_eloqua 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d2161d57974de45c6ec0cafeb73bd3d63da033fe
-  data.tar.gz: 3c3c1f880856690a453099e8e7934cddec9fd548
+  metadata.gz: 294f5ae81c518f15f02473db8460e6acf16bb07d
+  data.tar.gz: 9e830b5735f25b8c0f5734410fafe5eb7bf024a3
 SHA512:
-  metadata.gz: efd49b4be7c7220f681088d26c7496467e198b1bd47d0fcb22402cfbddf6f080218cabd2458bbb37011b970b8fc6d3d6ed7e30c219d9a3eb7ade64bbbc78f7c6
-  data.tar.gz: de16aa328329358215c2e38c170268fb8c7d359f786408308336206026b9e51b42cb340c6c5d146fb17f20be3862668f3e764483ed50adcc96aca1d7e62d37fd
+  metadata.gz: 1c4fbb45c71de382b763669b2f24fef92857d942313b2576d8a989527eafd65aebccb5728666759f019441e6dcaa7579b8fecc0e9ed222915befbf9cde46093a
+  data.tar.gz: bda53300869551d6ed5a2fefd12a733e251c7ecd4cacc6c1f8910abd349433d3eb38e0d23bc0280281894b7f68e66143bba31927b5191514e0a7684715d54196

data/lib/gooddata_eloqua/client.rb

@@ -11,11 +11,20 @@ require 'rest_client'
 class GoodDataEloqua::Client
 
   def initialize(config={})
+
     $global_id = SecureRandom.hex
     $client = config[:eloqua] || config[:client]
 
+    if config[:threading]
+      $threading = config[:threading]
+    else
+      $threading = true
+    end
+
     self.system_check
 
+    puts "#{Time.now} => Initialized job with session id: #{$global_id}."
+
   end
 
   def campaigns(config={})

data/lib/gooddata_eloqua/helpers/request.rb

@@ -65,19 +65,37 @@ class GoodDataEloqua::Request
       puts "#{Time.now} => #{iterations.length} pages queued for download."
       count = iterations.length
       mutex = Mutex.new
-      iterations.pmap { |page|
 
-        response = self.get_all_by_page(page)
-        print "\r#{Time.now} => Extracting IDs - Remaining Pages: #{count}\s\s\s"
+      if $threading
 
-        mutex.synchronize {
-          count -= 1
-          response['elements'].each { |element|
-            csv << [element['id']]
+        iterations.pmap { |page|
+
+          response = self.get_all_by_page(page)
+          print "\r#{Time.now} => Extracting IDs - Remaining Pages: #{count}\s\s\s"
+
+          mutex.synchronize {
+            count -= 1
+            response['elements'].each { |element|
+              csv << [element['id']]
+            }
           }
+
         }
+      else
+        iterations.each { |page|
 
-      }
+          response = self.get_all_by_page(page)
+          print "\r#{Time.now} => Extracting IDs - Remaining Pages: #{count}\s\s\s"
+
+          mutex.synchronize {
+            count -= 1
+            response['elements'].each { |element|
+              csv << [element['id']]
+            }
+          }
+
+        }
+      end
 
       csv.flush
       puts "#{Time.now} => #{count} IDs extracted."
@@ -103,25 +121,118 @@ class GoodDataEloqua::Request
     write_fd = IO.sysopen(file_name, 'w+')
     @csv = CSV.new(IO.new(write_fd))
 
+    read_csv = CSV.open(fd)
 
-    CSV.open(fd) do |read_csv|
+    ids = read_csv.map { |row|
+      row[0]
+    }
 
-      ids = read_csv.map { |row|
-        row[0]
-      }
+    return if ids.empty?
 
-      return if ids.empty?
+    cache = []
+    @headers = nil
+    set_headers = []
+    have_not_set_headers = false
+    count = 0
 
-      cache = []
-      headers = nil
-      count = 0
-      mutex = Mutex.new
+    if $threading
+      ids.pmap { |id| # <- Change this back to parallel
 
+        unless set_headers.empty?
+          puts "\nWAS ABLE TO SET HEADER\n"
+          @csv << set_headers.pop
+        end
 
-      ids.pmap { |id|
+        if cache.length > 2000
+          batch = cache.slice!(1..500)
+          batch.map { |row| @csv << row }
+        end
 
-        if cache.length > 1000
-          batch = cache.slice!(1..150)
+        count += 1
+        response = self.get_one_by_id(id)
+
+        case response['type']
+          when "Contact"
+            keys = 'fieldValues'
+          when "Campaign"
+            keys = 'elements'
+          when "Email"
+            keys = 'htmlContent'
+          when "Form"
+            keys = 'elements'
+          else
+            keys = 'elements'
+        end
+
+        if response.empty?
+
+          payload = [[id]]
+
+        else
+
+          if @headers == nil
+
+            begin
+              unless response[keys].empty?
+
+                element_keys = response[keys][0].keys.map { |key|
+                  "element_#{key}"
+                }
+
+              else
+                element_keys = []
+              end
+
+              @headers = response.keys + element_keys
+              set_headers << response.keys + element_keys
+
+            rescue NoMethodError
+              headers = [id]
+            end
+
+          end
+
+          if response[keys].is_a? Array
+            if response[keys].empty?
+              payload = [response.values]
+            else
+              payload = []
+
+              response[keys].each { |element|
+                payload << response.values + element.values
+              }
+            end
+          else
+            payload = [response]
+          end
+
+        end
+
+        ######### EDITING PAYLOAD TO REMOVE KEY ROW VALUES #####
+
+        payload = payload[0].map { |key_value|
+          if key_value.is_a? Array or key_value.is_a? Hash
+            key_value.to_json
+          else
+            key_value
+          end
+        }
+
+        cache << payload
+
+        print "\r#{Time.now} => Extracting Profiles - IDs Remaining: #{ids.length-count} Cache: #{cache.length}\s\s\s"
+
+      }
+    else
+      ids.each { |id| # <- Change this back to parallel
+
+        unless set_headers.empty?
+          puts "\nWAS ABLE TO SET HEADER\n"
+          @csv << set_headers.pop
+        end
+
+        if cache.length > 2000
+          batch = cache.slice!(1..500)
           batch.map { |row| @csv << row }
         end
 
@@ -147,7 +258,7 @@ class GoodDataEloqua::Request
 
         else
 
-          if headers == nil
+          if @headers == nil
 
             begin
               unless response[keys].empty?
@@ -160,7 +271,8 @@ class GoodDataEloqua::Request
                 element_keys = []
               end
 
-              headers = response.keys + element_keys
+              @headers = response.keys + element_keys
+              set_headers << response.keys + element_keys
 
             rescue NoMethodError
               headers = [id]
@@ -184,106 +296,57 @@ class GoodDataEloqua::Request
 
         end
 
+        ######### EDITING PAYLOAD TO REMOVE KEY ROW VALUES #####
+
+        payload = payload[0].map { |key_value|
+          if key_value.is_a? Array or key_value.is_a? Hash
+            key_value.to_json
+          else
+            key_value
+          end
+        }
+
         cache << payload
 
         print "\r#{Time.now} => Extracting Profiles - IDs Remaining: #{ids.length-count} Cache: #{cache.length}\s\s\s"
 
       }
 
-      puts "\n#{Time.now} => Threads complete!\n"
-      count = cache.length
+    end
+
+    puts "\n#{Time.now} => Threads complete!\n"
+    count = cache.length
+
+    if $threading
+
       cache.pmap { |row|
         print "\r#{Time.now} => Dumping pool to CSV #{count}\s\s\s\s "
         count -= 1
         @csv << row
-        @csv.flush
+
       }
 
-      puts "\n#{Time.now} => Flushing IO to \"#{file_name}\"\n"
+    else
+
+      cache.each { |row|
+        print "\r#{Time.now} => Dumping pool to CSV #{count}\s\s\s\s "
+        count -= 1
+        @csv << row
+
+      }
 
     end
 
+
+    @csv.flush
+
+    puts "\n#{Time.now} => Flushing IO to \"#{file_name}\"\n"
+
     puts "\n#{Time.now} => Complete: #{file_name}\n\n"
 
     file_name
 
-    #
-    #
-    # CSV.open(fd) do |read_csv|
-    #
-    #
-    #
-    #   puts "#{Time.now} => Extracted from CSV: #{ids.length}\n"
-    #
-    #   elapsed_times = []
-    #
-    #   loop do
-    #
-    #     batch = ids.slice!(1..20)
-    #
-    #     started_batch = Time.now
-    #
-    #     if elapsed_times.length > 5
-    #       estimated_time = elapsed_times.inject{ |sum, el| sum + el }.to_f / elapsed_times.size
-    #     else
-    #       estimated_time = 0
-    #     end
-    #
-    #     if (estimated_time/60) > 60 && estimated_time != 0
-    #       time_readable = "#{((estimated_time/60)/60).round(2)} hours"
-    #
-    #     elsif (estimated_time/60) < 60 && (estimated_time/60) > 1 && estimated_time != 0
-    #       time_readable = "#{(estimated_time/60).round(2)} minutes"
-    #
-    #     elsif (estimated_time/60) > 0 && estimated_time != 0
-    #       time_readable = "#{(estimated_time).round(2)} seconds"
-    #
-    #     else
-    #       time_readable = 'Estimating...'
-    #
-    #     end
-    #
-    #     break unless batch.length > 0 && pool.list.length
-    #
-    #     #
-    #     batch.each { |i|
-    #       pool.thread {
-    #         response = self.get_one_by_id(i)
-    #         csv << response.values
-    #         csv.flush
-    #         print "\r#{Time.now} => IDs Remaing: #{ids.length} - Complete: #{time_readable} - Thread Pool: #{pool.active_threads} \s\s\s\s"
-    #       }
-    #     }
-    #
-    #     ended_batch = Time.now
-    #
-    #     elapsed_time = ended_batch - started_batch
-    #     number_of_estimated_chunks = ids.length/20
-    #     estimated_batch_time = (elapsed_time*number_of_estimated_chunks).round
-    #
-    #     if elapsed_times.length < 10
-    #       elapsed_times << estimated_batch_time
-    #     elsif elapsed_times.length > 50
-    #       40.times do
-    #         elapsed_times.pop
-    #       end
-    #     end
-    #
-    #
-    #     if pool.active_threads > 21
-    #       sleep 3
-    #     end
-    #
-    #     break if ids.length == 0 && pool.list.length
-    #
-    #
-    #   end
-    #
-    #
-    #
-    #   puts "#{Time.now} => Complete. CSV written to \"./downloads/#{@session_id}_complete.csv\""
-    #
-    # end
+
 
 
 
@@ -312,33 +375,33 @@ class GoodDataEloqua::Request
 
     csv.flush
 
-    # pool = Pool.new
-    #
-    # while pool.running do
-    #
-    #   batch = iterations.slice!(1..20)
-    #
-    #   break unless batch
-    #
-    #   batch.each { |i|
-    #     pool.thread {
-    #
-    #
-    #       print "\r#{Time.now} => Pages #{iterations.length} of #{num_iterations} - Workers: #{pool.active_threads}\s"
-    #     }
-    #   }
-    #
-    #   if pool.active_threads > 20
-    #     sleep 2
-    #   end
-    #
-    #   if iterations.length == 0 && pool.active_threads == 0
-    #     pool.running = false
-    #   else
-    #     next
-    #   end
-    #
-    # end
+    pool = Pool.new
+
+    while pool.running do
+
+      batch = iterations.slice!(1..20)
+
+      break unless batch
+
+      batch.each { |i|
+        pool.thread {
+
+
+          print "\r#{Time.now} => Pages #{iterations.length} of #{num_iterations} - Workers: #{pool.active_threads}\s"
+        }
+      }
+
+      if pool.active_threads > 20
+        sleep 2
+      end
+
+      if iterations.length == 0 && pool.active_threads == 0
+        pool.running = false
+      else
+        next
+      end
+
+    end
 
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gooddata_eloqua
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Patrick McConlogue
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-05 00:00:00.000000000 Z
+date: 2015-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eloqua_api