remi 0.2.42 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. data/.yardopts +7 -0
  3. data/Gemfile +1 -1
  4. data/Gemfile.lock +13 -26
  5. data/README.md +1 -1
  6. data/features/step_definitions/remi_step.rb +33 -13
  7. data/features/sub_job_example.feature +24 -0
  8. data/features/sub_transform_example.feature +35 -0
  9. data/features/sub_transform_many_to_many.feature +49 -0
  10. data/features/support/env_app.rb +1 -1
  11. data/jobs/all_jobs_shared.rb +19 -16
  12. data/jobs/copy_source_job.rb +11 -9
  13. data/jobs/csv_file_target_job.rb +10 -9
  14. data/jobs/json_job.rb +18 -14
  15. data/jobs/metadata_job.rb +33 -28
  16. data/jobs/parameters_job.rb +14 -11
  17. data/jobs/sample_job.rb +106 -77
  18. data/jobs/sftp_file_target_job.rb +14 -13
  19. data/jobs/sub_job_example_job.rb +86 -0
  20. data/jobs/sub_transform_example_job.rb +43 -0
  21. data/jobs/sub_transform_many_to_many_job.rb +46 -0
  22. data/jobs/transforms/concatenate_job.rb +16 -12
  23. data/jobs/transforms/data_frame_sieve_job.rb +24 -19
  24. data/jobs/transforms/date_diff_job.rb +15 -11
  25. data/jobs/transforms/nvl_job.rb +16 -12
  26. data/jobs/transforms/parse_date_job.rb +17 -14
  27. data/jobs/transforms/partitioner_job.rb +27 -19
  28. data/jobs/transforms/prefix_job.rb +13 -10
  29. data/jobs/transforms/truncate_job.rb +14 -10
  30. data/jobs/transforms/truthy_job.rb +11 -8
  31. data/lib/remi.rb +25 -11
  32. data/lib/remi/data_frame.rb +4 -4
  33. data/lib/remi/data_frame/daru.rb +1 -37
  34. data/lib/remi/data_subject.rb +234 -48
  35. data/lib/remi/data_subjects/csv_file.rb +171 -0
  36. data/lib/remi/data_subjects/data_frame.rb +106 -0
  37. data/lib/remi/data_subjects/file_system.rb +115 -0
  38. data/lib/remi/data_subjects/local_file.rb +109 -0
  39. data/lib/remi/data_subjects/none.rb +31 -0
  40. data/lib/remi/data_subjects/postgres.rb +186 -0
  41. data/lib/remi/data_subjects/s3_file.rb +84 -0
  42. data/lib/remi/data_subjects/salesforce.rb +211 -0
  43. data/lib/remi/data_subjects/sftp_file.rb +196 -0
  44. data/lib/remi/data_subjects/sub_job.rb +50 -0
  45. data/lib/remi/dsl.rb +74 -0
  46. data/lib/remi/encoder.rb +45 -0
  47. data/lib/remi/extractor.rb +21 -0
  48. data/lib/remi/field_symbolizers.rb +1 -0
  49. data/lib/remi/job.rb +279 -113
  50. data/lib/remi/job/parameters.rb +90 -0
  51. data/lib/remi/job/sub_job.rb +35 -0
  52. data/lib/remi/job/transform.rb +165 -0
  53. data/lib/remi/loader.rb +22 -0
  54. data/lib/remi/monkeys/daru.rb +4 -0
  55. data/lib/remi/parser.rb +44 -0
  56. data/lib/remi/testing/business_rules.rb +17 -23
  57. data/lib/remi/testing/data_stub.rb +2 -2
  58. data/lib/remi/version.rb +1 -1
  59. data/remi.gemspec +3 -0
  60. data/spec/data_subject_spec.rb +475 -11
  61. data/spec/data_subjects/csv_file_spec.rb +69 -0
  62. data/spec/data_subjects/data_frame_spec.rb +52 -0
  63. data/spec/{extractor → data_subjects}/file_system_spec.rb +0 -0
  64. data/spec/{extractor → data_subjects}/local_file_spec.rb +0 -0
  65. data/spec/data_subjects/none_spec.rb +41 -0
  66. data/spec/data_subjects/postgres_spec.rb +80 -0
  67. data/spec/{extractor → data_subjects}/s3_file_spec.rb +0 -0
  68. data/spec/data_subjects/salesforce_spec.rb +117 -0
  69. data/spec/{extractor → data_subjects}/sftp_file_spec.rb +16 -0
  70. data/spec/data_subjects/sub_job_spec.rb +33 -0
  71. data/spec/encoder_spec.rb +38 -0
  72. data/spec/extractor_spec.rb +11 -0
  73. data/spec/fixtures/sf_bulk_helper_stubs.rb +443 -0
  74. data/spec/job/transform_spec.rb +257 -0
  75. data/spec/job_spec.rb +507 -0
  76. data/spec/loader_spec.rb +11 -0
  77. data/spec/parser_spec.rb +38 -0
  78. data/spec/sf_bulk_helper_spec.rb +117 -0
  79. data/spec/testing/data_stub_spec.rb +5 -3
  80. metadata +109 -27
  81. data/features/aggregate.feature +0 -42
  82. data/jobs/aggregate_job.rb +0 -31
  83. data/jobs/transforms/transform_jobs.rb +0 -4
  84. data/lib/remi/data_subject/csv_file.rb +0 -162
  85. data/lib/remi/data_subject/data_frame.rb +0 -52
  86. data/lib/remi/data_subject/postgres.rb +0 -134
  87. data/lib/remi/data_subject/salesforce.rb +0 -136
  88. data/lib/remi/data_subject/sftp_file.rb +0 -65
  89. data/lib/remi/extractor/file_system.rb +0 -92
  90. data/lib/remi/extractor/local_file.rb +0 -43
  91. data/lib/remi/extractor/s3_file.rb +0 -57
  92. data/lib/remi/extractor/sftp_file.rb +0 -83
  93. data/spec/data_subject/csv_file_spec.rb +0 -79
  94. data/spec/data_subject/data_frame.rb +0 -27
data/spec/extractor_spec.rb ADDED
@@ -0,0 +1,11 @@
1
+ require_relative 'remi_spec'
2
+
3
+ describe Remi::Extractor do
4
+ let(:extractor) { Extractor.new }
5
+
6
+ context '#extract' do
7
+ it 'has an extract method' do
8
+ expect(extractor).respond_to? :extract
9
+ end
10
+ end
11
+ end
data/spec/fixtures/sf_bulk_helper_stubs.rb ADDED
@@ -0,0 +1,443 @@
1
+ module Remi::SfBulkHelperStubs
2
+ extend self
3
+
4
+ def contact_query
5
+ <<-EOT
6
+ SELECT
7
+ Id,
8
+ Student_ID__c,
9
+ RecordTypeId,
10
+ FirstName,
11
+ LastName
12
+ FROM
13
+ Contact
14
+ WHERE
15
+ AccountId = '001G000000ncxb8IAA'
16
+ LIMIT 5
17
+ EOT
18
+ end
19
+
20
+ def contact_query_raw_result
21
+ {
22
+ "xmlns" => "http://www.force.com/2009/06/asyncapi/dataload",
23
+ "id" => [
24
+ "75016000004Ic1OAAS"
25
+ ],
26
+ "operation" => [
27
+ "query"
28
+ ],
29
+ "object" => [
30
+ "Contact"
31
+ ],
32
+ "createdById" => [
33
+ "005G0000005shkrIAA"
34
+ ],
35
+ "createdDate" => [
36
+ "2015-11-07T00:30:25.000Z"
37
+ ],
38
+ "systemModstamp" => [
39
+ "2015-11-07T00:30:25.000Z"
40
+ ],
41
+ "state" => [
42
+ "Closed"
43
+ ],
44
+ "concurrencyMode" => [
45
+ "Parallel"
46
+ ],
47
+ "contentType" => [
48
+ "XML"
49
+ ],
50
+ "numberBatchesQueued" => [
51
+ "0"
52
+ ],
53
+ "numberBatchesInProgress" => [
54
+ "0"
55
+ ],
56
+ "numberBatchesCompleted" => [
57
+ "1"
58
+ ],
59
+ "numberBatchesFailed" => [
60
+ "0"
61
+ ],
62
+ "numberBatchesTotal" => [
63
+ "1"
64
+ ],
65
+ "numberRecordsProcessed" => [
66
+ "5"
67
+ ],
68
+ "numberRetries" => [
69
+ "0"
70
+ ],
71
+ "apiVersion" => [
72
+ "32.0"
73
+ ],
74
+ "numberRecordsFailed" => [
75
+ "0"
76
+ ],
77
+ "totalProcessingTime" => [
78
+ "0"
79
+ ],
80
+ "apiActiveProcessingTime" => [
81
+ "0"
82
+ ],
83
+ "apexProcessingTime" => [
84
+ "0"
85
+ ],
86
+ "batches" => [
87
+ {
88
+ "xmlns" => "http://www.force.com/2009/06/asyncapi/dataload",
89
+ "id" => [
90
+ "751160000065e2BAAQ"
91
+ ],
92
+ "jobId" => [
93
+ "75016000004Ic1OAAS"
94
+ ],
95
+ "state" => [
96
+ "Completed"
97
+ ],
98
+ "createdDate" => [
99
+ "2015-11-07T00:30:25.000Z"
100
+ ],
101
+ "systemModstamp" => [
102
+ "2015-11-07T00:30:26.000Z"
103
+ ],
104
+ "numberRecordsProcessed" => [
105
+ "5"
106
+ ],
107
+ "numberRecordsFailed" => [
108
+ "0"
109
+ ],
110
+ "totalProcessingTime" => [
111
+ "0"
112
+ ],
113
+ "apiActiveProcessingTime" => [
114
+ "0"
115
+ ],
116
+ "apexProcessingTime" => [
117
+ "0"
118
+ ],
119
+ "response" => [
120
+ {
121
+ "xsi:type" => "sObject",
122
+ "type" => [
123
+ "Contact"
124
+ ],
125
+ "Id" => [
126
+ "003G000001cKYaUIA4",
127
+ "003G000001cKYaUIA4"
128
+ ],
129
+ "Student_ID__c" => [
130
+ "FJD385628"
131
+ ],
132
+ "RecordTypeId" => [
133
+ "012G0000000yEClIAM"
134
+ ],
135
+ "FirstName" => [
136
+ "Alicia"
137
+ ],
138
+ "LastName" => [
139
+ "Quantico"
140
+ ]
141
+ },
142
+ {
143
+ "xsi:type" => "sObject",
144
+ "type" => [
145
+ "Contact"
146
+ ],
147
+ "Id" => [
148
+ "003G000000cbYU0IAO",
149
+ "003G000000cbYU0IAO"
150
+ ],
151
+ "Student_ID__c" => [
152
+ "BHS0375638"
153
+ ],
154
+ "RecordTypeId" => [
155
+ "012G0000000yEClIAM"
156
+ ],
157
+ "FirstName" => [
158
+ "Jason"
159
+ ],
160
+ "LastName" => [
161
+ "Hockeymask"
162
+ ]
163
+ },
164
+ {
165
+ "xsi:type" => "sObject",
166
+ "type" => [
167
+ "Contact"
168
+ ],
169
+ "Id" => [
170
+ "003G000001cQYWyIAO",
171
+ "003G000001cQYWyIAO"
172
+ ],
173
+ "Student_ID__c" => [
174
+ "BHS83561365"
175
+ ],
176
+ "RecordTypeId" => [
177
+ "012G0000000yEClIAM"
178
+ ],
179
+ "FirstName" => [
180
+ "Marlin"
181
+ ],
182
+ "LastName" => [
183
+ "Jones"
184
+ ]
185
+ },
186
+ {
187
+ "xsi:type" => "sObject",
188
+ "type" => [
189
+ "Contact"
190
+ ],
191
+ "Id" => [
192
+ "003G000001cKYLrIAO",
193
+ "003G000001cKYLrIAO"
194
+ ],
195
+ "Student_ID__c" => [
196
+ "BHS3656616363"
197
+ ],
198
+ "RecordTypeId" => [
199
+ "012G0000000yEClIAM"
200
+ ],
201
+ "FirstName" => [
202
+ "Alberto"
203
+ ],
204
+ "LastName" => [
205
+ "Einsteinium"
206
+ ]
207
+ },
208
+ {
209
+ "xsi:type" => "sObject",
210
+ "type" => [
211
+ "Contact"
212
+ ],
213
+ "Id" => [
214
+ "002G000001cKYSzIAl",
215
+ "002G000001cKYSzIAl"
216
+ ],
217
+ "Student_ID__c" => [
218
+ "BHS338238855656"
219
+ ],
220
+ "RecordTypeId" => [
221
+ "012G0000000yEClIAM"
222
+ ],
223
+ "FirstName" => [
224
+ "Anthony"
225
+ ],
226
+ "LastName" => [
227
+ "Antimony"
228
+ ]
229
+ }
230
+ ]
231
+ }
232
+ ]
233
+ }
234
+ end
235
+
236
+ def contact_query_duplicated_result
237
+ [
238
+ { "Id"=>"003G000001cKYaUIA4", "Student_ID__c"=>"FJD385628" },
239
+ { "Id"=>"003G000000cbYU0IAO", "Student_ID__c"=>"FJD385628" },
240
+ { "Id"=>"003G000001cQYWyIAO", "Student_ID__c"=>"BHS83561365" },
241
+ { "Id"=>"003G000001cKYLrIAO", "Student_ID__c"=>"BHS3656616363" },
242
+ { "Id"=>"002G000001cKYSzIAl", "Student_ID__c"=>"BHS338238855656" }
243
+ ]
244
+ end
245
+
246
+ # Note, this is obfuscated data, do not run against the live production system.
247
+ def contact_update_data
248
+ [
249
+ {
250
+ 'Id' => '003G000002bKYOUIA4',
251
+ 'Student_ID__c' => 'FJD385628',
252
+ 'RecordTypeId' => '012G0000000yEClIAM',
253
+ 'FirstName' => 'Alicia',
254
+ 'LastName' => 'Quantico',
255
+ },
256
+ {
257
+ 'Id' => '003G000002caYU1IAO',
258
+ 'Student_ID__c' => 'BHS83561365',
259
+ 'RecordTypeId' => '012G0000000yEClIAM',
260
+ 'FirstName' => 'Jason',
261
+ 'LastName' => 'Hockeymask',
262
+ },
263
+ {
264
+ 'Id' => '003G000002cfyWyIAO',
265
+ 'Student_ID__c' => 'BHS3656616363',
266
+ 'RecordTypeId' => '012G0000000yEClIAM',
267
+ 'FirstName' => 'Alberto',
268
+ 'LastName' => 'Einsteinium',
269
+ }
270
+ ]
271
+ end
272
+
273
+ def contact_update_raw_result
274
+ {
275
+ "xmlns" => "http://www.force.com/2009/06/asyncapi/dataload",
276
+ "id" => [
277
+ "75016000004IfklAAC"
278
+ ],
279
+ "operation" => [
280
+ "update"
281
+ ],
282
+ "object" => [
283
+ "Contact"
284
+ ],
285
+ "createdById" => [
286
+ "005G0000005shkrIAA"
287
+ ],
288
+ "createdDate" => [
289
+ "2015-11-09T16:29:46.000Z"
290
+ ],
291
+ "systemModstamp" => [
292
+ "2015-11-09T16:29:46.000Z"
293
+ ],
294
+ "state" => [
295
+ "Closed"
296
+ ],
297
+ "concurrencyMode" => [
298
+ "Parallel"
299
+ ],
300
+ "contentType" => [
301
+ "XML"
302
+ ],
303
+ "numberBatchesQueued" => [
304
+ "0"
305
+ ],
306
+ "numberBatchesInProgress" => [
307
+ "0"
308
+ ],
309
+ "numberBatchesCompleted" => [
310
+ "1"
311
+ ],
312
+ "numberBatchesFailed" => [
313
+ "0"
314
+ ],
315
+ "numberBatchesTotal" => [
316
+ "1"
317
+ ],
318
+ "numberRecordsProcessed" => [
319
+ "3"
320
+ ],
321
+ "numberRetries" => [
322
+ "0"
323
+ ],
324
+ "apiVersion" => [
325
+ "32.0"
326
+ ],
327
+ "numberRecordsFailed" => [
328
+ "0"
329
+ ],
330
+ "totalProcessingTime" => [
331
+ "612"
332
+ ],
333
+ "apiActiveProcessingTime" => [
334
+ "501"
335
+ ],
336
+ "apexProcessingTime" => [
337
+ "379"
338
+ ],
339
+ "batches" => [
340
+ {
341
+ "xmlns" => "http://www.force.com/2009/06/asyncapi/dataload",
342
+ "id" => [
343
+ "751160000065kWlAAI"
344
+ ],
345
+ "jobId" => [
346
+ "75016000004IfklAAC"
347
+ ],
348
+ "state" => [
349
+ "Completed"
350
+ ],
351
+ "createdDate" => [
352
+ "2015-11-09T16:29:46.000Z"
353
+ ],
354
+ "systemModstamp" => [
355
+ "2015-11-09T16:29:47.000Z"
356
+ ],
357
+ "numberRecordsProcessed" => [
358
+ "3"
359
+ ],
360
+ "numberRecordsFailed" => [
361
+ "0"
362
+ ],
363
+ "totalProcessingTime" => [
364
+ "612"
365
+ ],
366
+ "apiActiveProcessingTime" => [
367
+ "501"
368
+ ],
369
+ "apexProcessingTime" => [
370
+ "379"
371
+ ],
372
+ "response" => [
373
+ {
374
+ "id" => [
375
+ "003G000002bKYOUIA4"
376
+ ],
377
+ "success" => [
378
+ "true"
379
+ ],
380
+ "created" => [
381
+ "false"
382
+ ]
383
+ },
384
+ {
385
+ "id" => [
386
+ "003G000002caYU1IAO"
387
+ ],
388
+ "success" => [
389
+ "true"
390
+ ],
391
+ "created" => [
392
+ "false"
393
+ ]
394
+ },
395
+ {
396
+ "id" => [
397
+ "003G000002cfyWyIAO"
398
+ ],
399
+ "success" => [
400
+ "true"
401
+ ],
402
+ "created" => [
403
+ "false"
404
+ ]
405
+ }
406
+ ]
407
+ }
408
+ ]
409
+ }
410
+ end
411
+
412
+ def unable_to_lock_row
413
+ {
414
+ "errors" => [
415
+ {
416
+ "message" => [
417
+ "unable to obtain exclusive access to this record"
418
+ ],
419
+ "statusCode" => [
420
+ "UNABLE_TO_LOCK_ROW"
421
+ ]
422
+ }
423
+ ],
424
+ "success" => [ "false" ],
425
+ "created" => [ "false" ]
426
+ }
427
+ end
428
+
429
+ def contact_update_with_fail_raw_result
430
+ update = JSON.parse(contact_update_raw_result.to_json)
431
+ update['numberRecordsFailed'] = ['2']
432
+ update['batches'][0]['response'][0] = unable_to_lock_row
433
+ update['batches'][0]['response'][2] = unable_to_lock_row
434
+
435
+ update
436
+ end
437
+
438
+ def contact_update_subsequent_success_raw_result
439
+ update = JSON.parse(contact_update_raw_result.to_json)
440
+ update['batches'][0]['response'].delete_at(1)
441
+ update
442
+ end
443
+ end