gooddata_eloqua 0.0.5 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dce7c75c7504f96b9124feb1a3f18eac14a6a4d7
-  data.tar.gz: e1ad512ea7480ae7b92bd23d67c9c5e00bbf168c
+  metadata.gz: 3bfdd680536bdc65c7003fdc816748ebf2006d9e
+  data.tar.gz: 62904c80d1487d0bb8b8553c62aaa5d49f25a84f
 SHA512:
-  metadata.gz: 701b92182ef8ab98a632b9105fdb7ed540f9312be7b25e494024da662e651e46bf2d95fdd8f886d3f19e4bbee423197c8e9dfde46d41bdaf8634cadc4d6c554c
-  data.tar.gz: cb49375f087136a0b554200610800e6ffbd1b8b05fb3417f362543794182a6ba24757bb778e2ad25a5fef68a76212fba3cba1bf7fbe716b283a34f4a571b149c
+  metadata.gz: 1a8f3d93a69619c86c4b4c0dda011d72bb2d55be269e618ee170628ed76bbb19979d94d993fef30c44331eb17886a45bb8c5d718d2259f9c1e320683ef31b8f2
+  data.tar.gz: 9e3a548f9ce3b9940973ed8e5a044375c9b5e9e924512ba73a689a66f1160b30b2e84d9d1ba2b6980917e946064a39011eaf75614ce148ee4d90af7bf2021896

data/lib/gooddata_eloqua/helpers/request.rb

@@ -29,9 +29,9 @@ class GoodDataEloqua::Request
 
   end
 
-  def write_all_ids_return_csv(file_name, config={})
-
+  def download_all_ids_return_csv(file_name, config={})
 
+    puts "#{Time.now} => Downloading IDs..."
     if file_name
       fd = IO.sysopen("./downloads/#{file_name}", 'w+')
     end
@@ -42,22 +42,47 @@ class GoodDataEloqua::Request
     pages = first_download['total'].to_i
     total = (pages/1000).round
 
-    total = 1 if total == 0
+    total = 1 if total == 0 || total < 1
 
     if total == 1
 
+      count = 0
       first_download['elements'].each { |record|
+        count += 1
         csv << [record['id']]
       }
 
+      puts "#{Time.now} => #{count} IDs extracted."
+      puts "#{Time.now} => Flushing IO to \"./downloads/#{file_name}\"\n"
       csv.flush
 
-      puts "\n#{Time.now} => Flushing IO (#{@session_csv}.id)..."
-
     else
 
-      self.get_all_distributed total, csv
-      puts "\n#{Time.now} => Flushing IO (#{@session_csv}.id)..."
+      iterations = []
+      total.times { |i| iterations << i }
+      iterations << total
+
+      puts "#{Time.now} => #{iterations.length} pages queued for download."
+      count = iterations.length
+      mutex = Mutex.new
+      iterations.pmap { |page|
+
+        response = self.get_all_by_page(page)
+        print "\r#{Time.now} => Extracting page: #{page} of #{count}\s\s\s"
+
+        mutex.synchronize {
+          count -= 1
+          response['elements'].each { |element|
+            csv << [element['id']]
+          }
+        }
+
+      }
+
+      csv.flush
+      puts "#{Time.now} => #{count} IDs extracted."
+
+      puts "\n#{Time.now} => Flushing IO to \"./downloads/#{file_name}\"\n"
 
     end
 
@@ -65,98 +90,197 @@ class GoodDataEloqua::Request
 
   end
 
-  def ids_to_eloqua_return_csv(csv, config = {})
+  def get_complete_profile_from_ids_csv(csv, config = {})
+
+    return unless File.exists? "./downloads/#{csv}"
 
+    puts "#{Time.now} => Extracting complete profile from IDs..."
     if csv
       fd = IO.sysopen("./downloads/#{csv}", 'r')
     end
 
-    started_method = Time.now
-
-    write_path = "./downloads/#{@session_id}_complete.csv"
-    write_fd = IO.sysopen(write_path, 'w+')
-    csv = CSV.new(IO.new(write_fd))
-    pool = Pool.new
-
+    file_name = "./downloads/#{@session_id}_complete_profile.csv"
+    write_fd = IO.sysopen(file_name, 'w+')
+    @csv = CSV.new(IO.new(write_fd))
+    mutex = Mutex.new
     CSV.open(fd) do |read_csv|
 
       ids = read_csv.map { |row|
         row[0]
       }
 
-      puts "#{Time.now} => Extracted from CSV: #{ids.length}\n"
+      return if ids.empty?
 
-      elapsed_times = []
+      headers = nil
+      rows = []
+      count = 0
 
-      loop do
+      def process_batch_of_rows batch
 
-        batch = ids.slice!(1..20)
+          batch.each { |row|
+            @csv << row
+          }
 
-        started_batch = Time.now
+      end
 
-        if elapsed_times.length > 5
-          estimated_time = elapsed_times.inject{ |sum, el| sum + el }.to_f / elapsed_times.size
-        else
-          estimated_time = 0
+      ids.pmap { |id|
+        count += 1
+        response = self.get_one_by_id(id)
+
+        case response['type']
+          when "Contact"
+            keys = 'fieldValues'
+          when "Campaign"
+            keys = 'elements'
+          when "Email"
+            keys = 'htmlContent'
+          when "Form"
+            keys = 'elements'
+          else
+            keys = 'elements'
         end
 
-        if (estimated_time/60) > 60 && estimated_time != 0
-          time_readable = "#{((estimated_time/60)/60).round(2)} hours"
-
-        elsif (estimated_time/60) < 60 && (estimated_time/60) > 1 && estimated_time != 0
-          time_readable = "#{(estimated_time/60).round(2)} minutes"
+        if response.empty?
 
-        elsif (estimated_time/60) > 0 && estimated_time != 0
-          time_readable = "#{(estimated_time).round(2)} seconds"
+          payload = [[id]]
 
         else
-          time_readable = 'Estimating...'
 
-        end
+          if headers == nil
 
-        break unless batch.length > 0 && pool.list.length
+            begin
+              unless response[keys].empty?
 
-        #
-        batch.each { |i|
-          pool.thread {
-            response = self.get_one_by_id(i)
-            csv << response.values
-            csv.flush
-            print "\r#{Time.now} => IDs Remaing: #{ids.length} - Complete: #{time_readable} - Thread Pool: #{pool.active_threads} \s\s\s\s"
-          }
-        }
+                element_keys = response[keys][0].keys.map { |key|
+                  "element_#{key}"
+                }
 
-        ended_batch = Time.now
+              else
+                element_keys = []
+              end
 
-        elapsed_time = ended_batch - started_batch
-        number_of_estimated_chunks = ids.length/20
-        estimated_batch_time = (elapsed_time*number_of_estimated_chunks).round
+              headers = response.keys + element_keys
+
+            rescue NoMethodError
+              headers = [id]
+            end
 
-        if elapsed_times.length < 10
-          elapsed_times << estimated_batch_time
-        elsif elapsed_times.length > 50
-          40.times do
-            elapsed_times.pop
           end
-        end
 
+          if response[keys].is_a? Array
+            if response[keys].empty?
+              payload = [response.values]
+            else
+              payload = []
+
+              response[keys].each { |element|
+                payload << response.values + element.values
+              }
+            end
+          else
+            payload = [response]
+          end
 
-        if pool.active_threads > 21
-          sleep 3
         end
 
-        break if ids.length == 0 && pool.list.length
+       rows << payload
+        print "\r#{Time.now} => IDs Remaining: #{ids.length-count} Pool: #{rows.length}-#{Thread.current}"
+      }
 
+      count = rows.length
+      rows.pmap { |row|
+        print "\r#{Time.now} => Dumping pool to CSV #{count}\s\s\s\s"
+        count -= 1
+        @csv << row
+      }
 
-      end
+      @csv.flush
 
+      puts "\n#{Time.now} => Flushing IO to \"#{file_name}\"\n"
 
+    end
 
-      puts "#{Time.now} => Complete. CSV written to \"./downloads/#{@session_id}_complete.csv\""
+    file_name
+
+    #
+    #
+    # CSV.open(fd) do |read_csv|
+    #
+    #
+    #
+    #   puts "#{Time.now} => Extracted from CSV: #{ids.length}\n"
+    #
+    #   elapsed_times = []
+    #
+    #   loop do
+    #
+    #     batch = ids.slice!(1..20)
+    #
+    #     started_batch = Time.now
+    #
+    #     if elapsed_times.length > 5
+    #       estimated_time = elapsed_times.inject{ |sum, el| sum + el }.to_f / elapsed_times.size
+    #     else
+    #       estimated_time = 0
+    #     end
+    #
+    #     if (estimated_time/60) > 60 && estimated_time != 0
+    #       time_readable = "#{((estimated_time/60)/60).round(2)} hours"
+    #
+    #     elsif (estimated_time/60) < 60 && (estimated_time/60) > 1 && estimated_time != 0
+    #       time_readable = "#{(estimated_time/60).round(2)} minutes"
+    #
+    #     elsif (estimated_time/60) > 0 && estimated_time != 0
+    #       time_readable = "#{(estimated_time).round(2)} seconds"
+    #
+    #     else
+    #       time_readable = 'Estimating...'
+    #
+    #     end
+    #
+    #     break unless batch.length > 0 && pool.list.length
+    #
+    #     #
+    #     batch.each { |i|
+    #       pool.thread {
+    #         response = self.get_one_by_id(i)
+    #         csv << response.values
+    #         csv.flush
+    #         print "\r#{Time.now} => IDs Remaing: #{ids.length} - Complete: #{time_readable} - Thread Pool: #{pool.active_threads} \s\s\s\s"
+    #       }
+    #     }
+    #
+    #     ended_batch = Time.now
+    #
+    #     elapsed_time = ended_batch - started_batch
+    #     number_of_estimated_chunks = ids.length/20
+    #     estimated_batch_time = (elapsed_time*number_of_estimated_chunks).round
+    #
+    #     if elapsed_times.length < 10
+    #       elapsed_times << estimated_batch_time
+    #     elsif elapsed_times.length > 50
+    #       40.times do
+    #         elapsed_times.pop
+    #       end
+    #     end
+    #
+    #
+    #     if pool.active_threads > 21
+    #       sleep 3
+    #     end
+    #
+    #     break if ids.length == 0 && pool.list.length
+    #
+    #
+    #   end
+    #
+    #
+    #
+    #   puts "#{Time.now} => Complete. CSV written to \"./downloads/#{@session_id}_complete.csv\""
+    #
+    # end
 
-    end
 
-    write_path
 
   end
 
@@ -167,36 +291,49 @@ class GoodDataEloqua::Request
     num_iterations.times { |i| iterations << i }
     iterations << num_iterations
 
-    pool = Pool.new
-
-    while pool.running do
-
-      batch = iterations.slice!(1..20)
+    mutex = Mutex.new
 
-      break unless batch
-
-      batch.each { |i|
-        pool.thread {
-          response = self.get_all_by_page(i)
-          response['elements'].each { |element|
-            csv << [element['id']]
-          }
-          csv.flush
-          print "\r#{Time.now} => Pages #{iterations.length} of #{num_iterations} - Workers: #{pool.active_threads}\s"
+    iterations.pmap { |page|
+      response = self.get_all_by_page(page)
+      puts "#{Time.now} => Extracting page: #{page}"
+      mutex.synchronize {
+        response['elements'].each { |element|
+          csv << [element['id']]
         }
-      }
 
-      if pool.active_threads > 20
-        sleep 2
-      end
-
-      if iterations.length == 0 && pool.active_threads == 0
-        pool.running = false
-      else
-        next
-      end
+      }
 
-    end
+    }
+
+    csv.flush
+
+    # pool = Pool.new
+    #
+    # while pool.running do
+    #
+    #   batch = iterations.slice!(1..20)
+    #
+    #   break unless batch
+    #
+    #   batch.each { |i|
+    #     pool.thread {
+    #
+    #
+    #       print "\r#{Time.now} => Pages #{iterations.length} of #{num_iterations} - Workers: #{pool.active_threads}\s"
+    #     }
+    #   }
+    #
+    #   if pool.active_threads > 20
+    #     sleep 2
+    #   end
+    #
+    #   if iterations.length == 0 && pool.active_threads == 0
+    #     pool.running = false
+    #   else
+    #     next
+    #   end
+    #
+    # end
 
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gooddata_eloqua
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.1.1
 platform: ruby
 authors:
 - Patrick McConlogue
@@ -28,16 +28,16 @@ dependencies:
   name: gooddata_s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement