gooddata_eloqua 0.1.2 → 0.1.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 068253414a851eb124da89d9e50c1c26e969a058
4
- data.tar.gz: a4fe093709e9d192665c0d55759ecfbb2ed64971
3
+ metadata.gz: d2161d57974de45c6ec0cafeb73bd3d63da033fe
4
+ data.tar.gz: 3c3c1f880856690a453099e8e7934cddec9fd548
5
5
  SHA512:
6
- metadata.gz: eaa610a62bbe5af34bd3db3946c8c049440c11e5f354ef9649792c876105c2a55af84051166d8ab74cf1e646d5dbea7ce8077fe4349b1ce3f4005af2fc94ccad
7
- data.tar.gz: b1f0f5dc3624baf0dcbe0fade98a30fddaa21bdca4a22b0ca6b19ccdfcb983e093e5ad58e96da6e56aa32841f28ddb1e225884a69d95e876b3f9865351c52c22
6
+ metadata.gz: efd49b4be7c7220f681088d26c7496467e198b1bd47d0fcb22402cfbddf6f080218cabd2458bbb37011b970b8fc6d3d6ed7e30c219d9a3eb7ade64bbbc78f7c6
7
+ data.tar.gz: de16aa328329358215c2e38c170268fb8c7d359f786408308336206026b9e51b42cb340c6c5d146fb17f20be3862668f3e764483ed50adcc96aca1d7e62d37fd
@@ -68,7 +68,7 @@ class GoodDataEloqua::Request
68
68
  iterations.pmap { |page|
69
69
 
70
70
  response = self.get_all_by_page(page)
71
- print "\r#{Time.now} => Extracting page: #{page} of #{count}\s\s\s"
71
+ print "\r#{Time.now} => Extracting IDs - Remaining Pages: #{count}\s\s\s"
72
72
 
73
73
  mutex.synchronize {
74
74
  count -= 1
@@ -102,7 +102,8 @@ class GoodDataEloqua::Request
102
102
  file_name = "./downloads/#{@session_id}_complete_profile.csv"
103
103
  write_fd = IO.sysopen(file_name, 'w+')
104
104
  @csv = CSV.new(IO.new(write_fd))
105
- mutex = Mutex.new
105
+
106
+
106
107
  CSV.open(fd) do |read_csv|
107
108
 
108
109
  ids = read_csv.map { |row|
@@ -111,19 +112,19 @@ class GoodDataEloqua::Request
111
112
 
112
113
  return if ids.empty?
113
114
 
115
+ cache = []
114
116
  headers = nil
115
- rows = []
116
117
  count = 0
118
+ mutex = Mutex.new
117
119
 
118
- def process_batch_of_rows batch
119
120
 
120
- batch.each { |row|
121
- @csv << row
122
- }
121
+ ids.pmap { |id|
123
122
 
124
- end
123
+ if cache.length > 1000
124
+ batch = cache.slice!(1..150)
125
+ batch.map { |row| @csv << row }
126
+ end
125
127
 
126
- ids.pmap { |id|
127
128
  count += 1
128
129
  response = self.get_one_by_id(id)
129
130
 
@@ -183,23 +184,27 @@ class GoodDataEloqua::Request
183
184
 
184
185
  end
185
186
 
186
- rows << payload
187
- print "\r#{Time.now} => IDs Remaining: #{ids.length-count} Pool: #{rows.length}-#{Thread.current}"
187
+ cache << payload
188
+
189
+ print "\r#{Time.now} => Extracting Profiles - IDs Remaining: #{ids.length-count} Cache: #{cache.length}\s\s\s"
190
+
188
191
  }
189
192
 
190
- count = rows.length
191
- rows.pmap { |row|
192
- print "\r#{Time.now} => Dumping pool to CSV #{count}\s\s\s\s"
193
+ puts "\n#{Time.now} => Threads complete!\n"
194
+ count = cache.length
195
+ cache.pmap { |row|
196
+ print "\r#{Time.now} => Dumping pool to CSV #{count}\s\s\s\s "
193
197
  count -= 1
194
198
  @csv << row
199
+ @csv.flush
195
200
  }
196
201
 
197
- @csv.flush
198
-
199
202
  puts "\n#{Time.now} => Flushing IO to \"#{file_name}\"\n"
200
203
 
201
204
  end
202
205
 
206
+ puts "\n#{Time.now} => Complete: #{file_name}\n\n"
207
+
203
208
  file_name
204
209
 
205
210
  #
@@ -295,7 +300,7 @@ class GoodDataEloqua::Request
295
300
 
296
301
  iterations.pmap { |page|
297
302
  response = self.get_all_by_page(page)
298
- puts "#{Time.now} => Extracting page: #{page}"
303
+ puts "#{Time.now} => Extracting IDs from page: #{page}"
299
304
  mutex.synchronize {
300
305
  response['elements'].each { |element|
301
306
  csv << [element['id']]
@@ -413,4 +418,5 @@ class GoodDataEloqua::Request
413
418
  FileUtils.rm_f("/downloads/#{@session_csv}")
414
419
  end
415
420
 
421
+
416
422
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gooddata_eloqua
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Patrick McConlogue