gooddata_eloqua 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 068253414a851eb124da89d9e50c1c26e969a058
-  data.tar.gz: a4fe093709e9d192665c0d55759ecfbb2ed64971
+  metadata.gz: d2161d57974de45c6ec0cafeb73bd3d63da033fe
+  data.tar.gz: 3c3c1f880856690a453099e8e7934cddec9fd548
 SHA512:
-  metadata.gz: eaa610a62bbe5af34bd3db3946c8c049440c11e5f354ef9649792c876105c2a55af84051166d8ab74cf1e646d5dbea7ce8077fe4349b1ce3f4005af2fc94ccad
-  data.tar.gz: b1f0f5dc3624baf0dcbe0fade98a30fddaa21bdca4a22b0ca6b19ccdfcb983e093e5ad58e96da6e56aa32841f28ddb1e225884a69d95e876b3f9865351c52c22
+  metadata.gz: efd49b4be7c7220f681088d26c7496467e198b1bd47d0fcb22402cfbddf6f080218cabd2458bbb37011b970b8fc6d3d6ed7e30c219d9a3eb7ade64bbbc78f7c6
+  data.tar.gz: de16aa328329358215c2e38c170268fb8c7d359f786408308336206026b9e51b42cb340c6c5d146fb17f20be3862668f3e764483ed50adcc96aca1d7e62d37fd

data/lib/gooddata_eloqua/helpers/request.rb

@@ -68,7 +68,7 @@ class GoodDataEloqua::Request
       iterations.pmap { |page|
 
         response = self.get_all_by_page(page)
-        print "\r#{Time.now} => Extracting page: #{page} of #{count}\s\s\s"
+        print "\r#{Time.now} => Extracting IDs - Remaining Pages: #{count}\s\s\s"
 
         mutex.synchronize {
           count -= 1
@@ -102,7 +102,8 @@ class GoodDataEloqua::Request
     file_name = "./downloads/#{@session_id}_complete_profile.csv"
     write_fd = IO.sysopen(file_name, 'w+')
     @csv = CSV.new(IO.new(write_fd))
-    mutex = Mutex.new
+
+
     CSV.open(fd) do |read_csv|
 
       ids = read_csv.map { |row|
@@ -111,19 +112,19 @@ class GoodDataEloqua::Request
 
       return if ids.empty?
 
+      cache = []
       headers = nil
-      rows = []
       count = 0
+      mutex = Mutex.new
 
-      def process_batch_of_rows batch
 
-          batch.each { |row|
-            @csv << row
-          }
+      ids.pmap { |id|
 
-      end
+        if cache.length > 1000
+          batch = cache.slice!(1..150)
+          batch.map { |row| @csv << row }
+        end
 
-      ids.pmap { |id|
         count += 1
         response = self.get_one_by_id(id)
 
@@ -183,23 +184,27 @@ class GoodDataEloqua::Request
 
         end
 
-       rows << payload
-        print "\r#{Time.now} => IDs Remaining: #{ids.length-count} Pool: #{rows.length}-#{Thread.current}"
+        cache << payload
+
+        print "\r#{Time.now} => Extracting Profiles - IDs Remaining: #{ids.length-count} Cache: #{cache.length}\s\s\s"
+
       }
 
-      count = rows.length
-      rows.pmap { |row|
-        print "\r#{Time.now} => Dumping pool to CSV #{count}\s\s\s\s"
+      puts "\n#{Time.now} => Threads complete!\n"
+      count = cache.length
+      cache.pmap { |row|
+        print "\r#{Time.now} => Dumping pool to CSV #{count}\s\s\s\s "
         count -= 1
         @csv << row
+        @csv.flush
       }
 
-      @csv.flush
-
       puts "\n#{Time.now} => Flushing IO to \"#{file_name}\"\n"
 
     end
 
+    puts "\n#{Time.now} => Complete: #{file_name}\n\n"
+
     file_name
 
     #
@@ -295,7 +300,7 @@ class GoodDataEloqua::Request
 
     iterations.pmap { |page|
       response = self.get_all_by_page(page)
-      puts "#{Time.now} => Extracting page: #{page}"
+      puts "#{Time.now} => Extracting IDs from page: #{page}"
       mutex.synchronize {
         response['elements'].each { |element|
           csv << [element['id']]
@@ -413,4 +418,5 @@ class GoodDataEloqua::Request
     FileUtils.rm_f("/downloads/#{@session_csv}")
   end
 
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gooddata_eloqua
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Patrick McConlogue