remi 0.2.17 → 0.2.18
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +2 -2
- data/features/formulas.feature +2 -1
- data/features/sample_job.feature +4 -4
- data/features/step_definitions/remi_step.rb +175 -71
- data/features/transforms/truncate.feature +14 -0
- data/jobs/transforms/truncate_job.rb +19 -0
- data/lib/remi/cucumber/business_rules.rb +32 -14
- data/lib/remi/transform.rb +6 -0
- data/lib/remi/version.rb +1 -1
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7c112dfed8a5515c7c839e41c038181c79364504
|
4
|
+
data.tar.gz: a7185273e9bb567e70a83bf09a8670cf85d4d223
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 81645e029daa594ec66f7b109443f1337a1e2af0c5d85daf0f54d6e3f6254ad4c02ba637b31b96eb487aa2f3c1a8df535b90f7f67b29b2f61eb291d8bc052443
|
7
|
+
data.tar.gz: 6a700425dc41377ffa5e56aa34a46eb4c7d9d9f9e9d37c48c631379f0cf3336e8fab1188a43d7b908b39dbbfa154176bc0473427932c92b1823795e4504897aa
|
data/Gemfile.lock
CHANGED
data/features/formulas.feature
CHANGED
@@ -72,4 +72,5 @@ Feature: This tests the creation of example records.
|
|
72
72
|
And the source field 'Some Date'
|
73
73
|
And the source field 'Some String'
|
74
74
|
And the target field 'Combination'
|
75
|
-
Then the target field is a concatenation of 'Some Date' and 'Some String', delimited by "-"
|
75
|
+
Then the target field 'Combination' is a concatenation of the source fields 'Some Date' and 'Some String', delimited by "-"
|
76
|
+
Then the target field 'Combination' is a concatenation of the source fields, delimited by "-"
|
data/features/sample_job.feature
CHANGED
@@ -195,12 +195,12 @@ Feature: This is a sample feature file.
|
|
195
195
|
Then the target field '<Target Field>' is populated from the source field using the format "<Target Format>"
|
196
196
|
|
197
197
|
When the source field is blank
|
198
|
-
Then the target field '<Target Field>' is populated with "<If Blank>"
|
198
|
+
Then the target field '<Target Field>' is populated with "<If Blank>"
|
199
199
|
|
200
200
|
Examples:
|
201
|
-
| Source Field | Source Format | Target Field | Target Format | If Blank
|
202
|
-
| Applied Date | %m/%d/%Y | Applied_Date__c | %Y-%m-%d | *Today
|
203
|
-
| Birthdate | %m/%d/%Y | Birthdate | %Y-%m-%d |
|
201
|
+
| Source Field | Source Format | Target Field | Target Format | If Blank |
|
202
|
+
| Applied Date | %m/%d/%Y | Applied_Date__c | %Y-%m-%d | *Today: %Y-%m-%d* |
|
203
|
+
| Birthdate | %m/%d/%Y | Birthdate | %Y-%m-%d | |
|
204
204
|
|
205
205
|
|
206
206
|
Scenario: Populating School__c
|
@@ -106,7 +106,7 @@ Given /^the (source|target) file contains at least the following headers in no p
|
|
106
106
|
field = row.first
|
107
107
|
step "the #{st} field '#{field}'"
|
108
108
|
end
|
109
|
-
expect(@brt.send(st.to_sym).data_obj.fields.keys).to include(*@brt.send(st.to_sym).fields.
|
109
|
+
expect(@brt.send(st.to_sym).data_obj.fields.keys).to include(*@brt.send(st.to_sym).fields.field_names)
|
110
110
|
end
|
111
111
|
|
112
112
|
Given /^the (source|target) file contains all of the following headers in this order:$/ do |st, table|
|
@@ -116,7 +116,7 @@ Given /^the (source|target) file contains all of the following headers in this o
|
|
116
116
|
end
|
117
117
|
|
118
118
|
@brt.run_transforms if st == 'target'
|
119
|
-
expect(@brt.send(st.to_sym).data_obj.fields.keys).to eq @brt.send(st.to_sym).fields.
|
119
|
+
expect(@brt.send(st.to_sym).data_obj.fields.keys).to eq @brt.send(st.to_sym).fields.field_names
|
120
120
|
end
|
121
121
|
|
122
122
|
|
@@ -130,29 +130,31 @@ Given /^the source field '([^']+)'$/ do |source_field_name|
|
|
130
130
|
@brt.sources.add_field(source_field_name)
|
131
131
|
end
|
132
132
|
|
133
|
-
Given /^the source field has the value "([^"]*)"$/ do |
|
134
|
-
|
133
|
+
Given /^the source field '([^']+)' (?:has|is set to) the value "([^"]*)"$/ do |source_field, value|
|
134
|
+
step "the source field '#{source_field}'"
|
135
|
+
|
136
|
+
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
137
|
+
@brt.sources[source_name].fields[source_field_name].value = Remi::BusinessRules::ParseFormula.parse(value)
|
135
138
|
end
|
136
139
|
|
137
|
-
|
138
|
-
@brt.
|
140
|
+
Given /^the source field (?:has|is set to) the value "([^"]*)"$/ do |value|
|
141
|
+
@brt.sources.fields.each do |field|
|
142
|
+
step "the source field '#{field.full_name}' is set to the value \"#{value}\""
|
143
|
+
end
|
139
144
|
end
|
140
145
|
|
141
|
-
When /^the source field '(
|
146
|
+
When /^the source field '([^']+)' (?:has an empty value|is blank)$/ do |source_field|
|
142
147
|
step "the source field '#{source_field}'"
|
148
|
+
|
143
149
|
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
144
|
-
@brt.sources[source_name].fields[source_field_name].value = ''
|
145
|
-
end
|
146
150
|
|
147
|
-
|
148
|
-
step "the source field '#{source_field}'"
|
149
|
-
@brt.source.fields[source_field].value = Remi::BusinessRules::ParseFormula.parse(value)
|
151
|
+
@brt.sources[source_name].fields[source_field_name].value = ''
|
150
152
|
end
|
151
153
|
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
154
|
+
When /^the source field (?:has an empty value|is blank)$/ do
|
155
|
+
@brt.sources.fields.each do |field|
|
156
|
+
step "the source field '#{field.full_name}' is blank"
|
157
|
+
end
|
156
158
|
end
|
157
159
|
|
158
160
|
Given /^the source field '(.+:.+)' (?:has|is set to) the value in the source field '(.+:.+)'$/ do |source_field, other_source_field|
|
@@ -178,8 +180,17 @@ Given /^the source data are tied through the fields '(.+:.+)' and '(.+:.+)'$/ do
|
|
178
180
|
step "the source field '#{other_source_field}' is set to the value in the source field '#{source_field}'"
|
179
181
|
end
|
180
182
|
|
183
|
+
Given /^the source field '([^']+)' is parsed with the date format "([^"]*)"$/ do |source_field, date_format|
|
184
|
+
step "the source field '#{source_field}'"
|
185
|
+
|
186
|
+
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
187
|
+
expect(@brt.sources[source_name].fields[source_field_name].metadata[:format]).to eq date_format
|
188
|
+
end
|
189
|
+
|
181
190
|
Given /^the source field is parsed with the date format "([^"]*)"$/ do |date_format|
|
182
|
-
|
191
|
+
@brt.sources.fields.each do |field|
|
192
|
+
step "the source field '#{field.full_name}' is parsed with the date format \"#{date_format}\""
|
193
|
+
end
|
183
194
|
end
|
184
195
|
|
185
196
|
Given /^the source field is a valid email address$/ do
|
@@ -200,55 +211,66 @@ Given /^the target field '([^']+)'$/ do |arg|
|
|
200
211
|
@brt.targets.add_field(arg)
|
201
212
|
end
|
202
213
|
|
203
|
-
Then /^the target field '(
|
204
|
-
step "the target field '#{arg}'"
|
205
|
-
step "the target field is copied from the source field"
|
206
|
-
end
|
207
|
-
|
208
|
-
Then /^the target field is copied from the source field$/ do
|
209
|
-
@brt.run_transforms
|
210
|
-
expect(@brt.target.field.value).to eq (@brt.source.field.value)
|
211
|
-
end
|
212
|
-
|
213
|
-
Then /^the target field '(.+)' is copied from the source field '(.+:.+)'$/ do |target_field, source_field|
|
214
|
+
Then /^the target field '([^']+)' is copied from the source field '([^']+)'$/ do |target_field, source_field|
|
214
215
|
step "the target field '#{target_field}'"
|
215
216
|
step "the source field '#{source_field}'"
|
216
217
|
|
217
218
|
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
219
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
218
220
|
|
219
221
|
@brt.run_transforms
|
220
|
-
|
222
|
+
Array(target_names).each do |target_name|
|
223
|
+
expect(@brt.targets[target_name].fields[target_field_name].value).to eq (@brt.sources[source_name].fields[source_field_name].value)
|
224
|
+
end
|
221
225
|
end
|
222
226
|
|
223
|
-
Then /^the target field '(
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
@brt.run_transforms
|
228
|
-
expect(@brt.target.field.value).to eq (@brt.source.fields[source_field].value)
|
227
|
+
Then /^the target field '([^']+)' is copied from the source field$/ do |target_field|
|
228
|
+
@brt.sources.fields.each do |source_field|
|
229
|
+
step "the target field '#{target_field}' is copied from the source field '#{source_field.full_name}'"
|
230
|
+
end
|
229
231
|
end
|
230
232
|
|
231
|
-
Then /^the target field is
|
232
|
-
@brt.
|
233
|
-
|
233
|
+
Then /^the target field is copied from the source field$/ do
|
234
|
+
@brt.targets.fields.each do |target_field|
|
235
|
+
@brt.sources.fields.each do |source_field|
|
236
|
+
step "the target field '#{target_field.full_name}' is copied from the source field '#{source_field.full_name}'"
|
237
|
+
end
|
238
|
+
end
|
234
239
|
end
|
235
240
|
|
236
|
-
Then /^the target field '(
|
241
|
+
Then /^the target field '([^']+)' is (?:set to the value|populated with) "([^"]*)"$/ do |target_field, value|
|
237
242
|
expect_cucumber {
|
243
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
244
|
+
|
238
245
|
expect {
|
239
|
-
|
240
|
-
|
246
|
+
Array(target_names).each do |target_name|
|
247
|
+
@brt.targets[target_name].add_field(target_field_name)
|
248
|
+
end
|
249
|
+
@brt.run_transforms
|
241
250
|
}.not_to raise_error
|
242
|
-
|
251
|
+
Array(target_names).each do |target_name|
|
252
|
+
expect(@brt.targets[target_name].fields[target_field_name].values.uniq).to eq [Remi::BusinessRules::ParseFormula.parse(value)]
|
253
|
+
end
|
243
254
|
}
|
244
255
|
end
|
245
256
|
|
246
|
-
Then /^the target field
|
257
|
+
Then /^the target field is (?:set to the value|populated with) "([^"]*)"$/ do |value|
|
258
|
+
@brt.targets.fields.each do |field|
|
259
|
+
step "the target field '#{field.full_name}' is populated with \"#{value}\""
|
260
|
+
end
|
261
|
+
end
|
262
|
+
|
263
|
+
|
264
|
+
Then /^the target field '(.+)' has a value in the list "([^"]*)"$/ do |target_field, list|
|
247
265
|
step "the target field '#{target_field}'"
|
248
266
|
|
267
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
268
|
+
|
249
269
|
list_array = list.split(',').map(&:strip)
|
250
270
|
@brt.run_transforms
|
251
|
-
|
271
|
+
Array(target_names).each do |target_name|
|
272
|
+
expect(@brt.targets[target_name].fields[target_field].values.uniq & list_array).to include(*@brt.targets[target_name].fields[target_field].values.uniq)
|
273
|
+
end
|
252
274
|
end
|
253
275
|
|
254
276
|
|
@@ -284,31 +306,82 @@ end
|
|
284
306
|
|
285
307
|
### Transforms
|
286
308
|
|
309
|
+
Then /^the target field '([^']+)' is a concatenation of the source fields '(.+)', delimited by "([^"]*)"$/ do |target_field, source_field_list, delimiter|
|
310
|
+
source_fields = "'#{source_field_list}'".gsub(' and ', ', ').split(',').map do |field_with_quotes|
|
311
|
+
full_field_name = field_with_quotes.match(/'(.+)'/)[1]
|
312
|
+
|
313
|
+
source_name, field_name = @brt.sources.parse_full_field(full_field_name)
|
314
|
+
{ full_field_name: full_field_name, source: source_name, field: field_name }
|
315
|
+
end
|
316
|
+
|
317
|
+
concatenated_source = source_fields.map do |field|
|
318
|
+
step "the source field '#{field[:full_field_name]}'"
|
319
|
+
@brt.sources[field[:source]].fields[field[:field]].values.uniq
|
320
|
+
end.join(delimiter)
|
321
|
+
|
322
|
+
step "the target field '#{target_field}'"
|
323
|
+
|
324
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
325
|
+
|
326
|
+
@brt.run_transforms
|
327
|
+
Array(target_names).each do |target_name|
|
328
|
+
expect(@brt.targets[target_name].fields[target_field_name].values.uniq).to eq Array(concatenated_source)
|
329
|
+
end
|
330
|
+
end
|
331
|
+
|
332
|
+
Then /^the target field '([^']+)' is a concatenation of the source fields, delimited by "([^"]*)"$/ do |target_field, delimiter|
|
333
|
+
source_field_list = @brt.sources.fields.map do |field|
|
334
|
+
"'#{field.full_name}'"
|
335
|
+
end.join(',')
|
336
|
+
|
337
|
+
step "the target field '#{target_field}' is a concatenation of the source fields #{source_field_list}, delimited by \"#{delimiter}\""
|
338
|
+
end
|
339
|
+
|
287
340
|
Then /^the target field is a concatenation of the source fields, delimited by "([^"]*)"$/ do |delimiter|
|
288
|
-
|
289
|
-
|
341
|
+
@brt.targets.fields.each do |target_field|
|
342
|
+
step "the target field '#{target_field.full_name}' is a concatenation of the source fields, delimited by \"#{delimiter}\""
|
290
343
|
end
|
344
|
+
end
|
345
|
+
|
291
346
|
|
347
|
+
Then /^the target field '([^']+)' is a concatenation of "([^"]*)" and '(.+)', delimited by "([^"]*)"$/ do |target_field, constant, source_field, delimiter|
|
348
|
+
step "the target field '#{target_field}'"
|
349
|
+
|
350
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
351
|
+
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
352
|
+
|
353
|
+
expected_value = [constant, @brt.sources[source_name].fields[source_field_name].value].join(delimiter)
|
292
354
|
@brt.run_transforms
|
293
|
-
|
355
|
+
|
356
|
+
Array(target_names).each do |target_name|
|
357
|
+
expect(@brt.targets[target_name].fields[target_field_name].values.uniq).to eq Array(expected_value)
|
358
|
+
end
|
294
359
|
end
|
295
360
|
|
296
|
-
Then /^the target field is a concatenation of '(.+)' and
|
297
|
-
|
361
|
+
Then /^the target field '([^']+)' is a concatenation of '(.+)' and "([^"]*)", delimited by "([^"]*)"$/ do |target_field, source_field, constant, delimiter|
|
362
|
+
step "the target field '#{target_field}'"
|
363
|
+
|
364
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
365
|
+
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
366
|
+
|
367
|
+
expected_value = [@brt.sources[source_name].fields[source_field_name].value, constant].join(delimiter)
|
298
368
|
@brt.run_transforms
|
299
|
-
|
369
|
+
|
370
|
+
Array(target_names).each do |target_name|
|
371
|
+
expect(@brt.targets[target_name].fields[target_field_name].values.uniq).to eq Array(expected_value)
|
372
|
+
end
|
300
373
|
end
|
301
374
|
|
302
375
|
Then /^the target field is a concatenation of "([^"]*)" and '(.+)', delimited by "([^"]*)"$/ do |constant, source_field, delimiter|
|
303
|
-
|
304
|
-
|
305
|
-
|
376
|
+
@brt.targets.fields.each do |target_field|
|
377
|
+
step "the target field '#{target_field.full_name}' is a concatenation of \"#{constant}\" and '#{source_field}', delimited by \"#{delimiter}\""
|
378
|
+
end
|
306
379
|
end
|
307
380
|
|
308
381
|
Then /^the target field is a concatenation of '(.+)' and "([^"]*)", delimited by "([^"]*)"$/ do |source_field, constant, delimiter|
|
309
|
-
|
310
|
-
|
311
|
-
|
382
|
+
@brt.targets.fields.each do |target_field|
|
383
|
+
step "the target field '#{target_field.full_name}' is a concatenation of '#{source_field}' and \"#{constant}\", delimited by \"#{delimiter}\""
|
384
|
+
end
|
312
385
|
end
|
313
386
|
|
314
387
|
Then /^the source field is prefixed with "([^"]*)" and loaded into the target field$/ do |prefix|
|
@@ -317,25 +390,27 @@ Then /^the source field is prefixed with "([^"]*)" and loaded into the target fi
|
|
317
390
|
expect(@brt.target.field.value).to eq prefixed_source
|
318
391
|
end
|
319
392
|
|
320
|
-
Then /^the target field '(
|
321
|
-
|
393
|
+
Then /^the target field '([^']+)' is populated from the source field '([^']+)' using the format "([^"]*)"$/ do |target_field, source_field, target_format|
|
394
|
+
step "the source field '#{source_field}'"
|
395
|
+
step "the target field '#{target_field}'"
|
396
|
+
|
397
|
+
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
398
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
399
|
+
|
400
|
+
source_format = @brt.sources[source_name].fields[source_field_name].metadata[:format]
|
322
401
|
source_reformatted = Remi::Transform[:format_date].(from_fmt: source_format, to_fmt: target_format)
|
323
|
-
.call(@brt.
|
402
|
+
.call(@brt.sources[source_name].fields[source_field_name].value)
|
324
403
|
|
325
|
-
step "the target field '#{target_field}'"
|
326
404
|
@brt.run_transforms
|
327
|
-
|
405
|
+
target_names.each do |target_name|
|
406
|
+
expect(@brt.targets[target_name].fields[target_field_name].value).to eq source_reformatted
|
407
|
+
end
|
328
408
|
end
|
329
409
|
|
330
|
-
Then /^the target field '(
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
.call(target_value_source_format)
|
335
|
-
|
336
|
-
step "the target field '#{target_field}'"
|
337
|
-
@brt.run_transforms
|
338
|
-
expect(@brt.target.field.value).to eq target_reformatted
|
410
|
+
Then /^the target field '([^']+)' is populated from the source field using the format "([^"]*)"$/ do |target_field, target_format|
|
411
|
+
@brt.sources.fields.each do |source_field|
|
412
|
+
step "the target field '#{target_field}' is populated from the source field '#{source_field.full_name}' using the format \"#{target_format}\""
|
413
|
+
end
|
339
414
|
end
|
340
415
|
|
341
416
|
Then /^the target field '(.+)' is the first non-blank value from source fields '(.+)'$/ do |target_field_name, source_field_list|
|
@@ -374,6 +449,35 @@ Then /^the target field is copied from the source field, but commas have been re
|
|
374
449
|
expect(@brt.target.field.value).to eq source_field_value.gsub(/,/, '.')
|
375
450
|
end
|
376
451
|
|
452
|
+
Then /^the target field '([^']+)' contains a unique value matching the pattern \/(.*)\/$/ do |target_field, pattern|
|
453
|
+
step "the target field '#{target_field}'"
|
454
|
+
|
455
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
456
|
+
regex_pattern = Regexp.new(pattern)
|
457
|
+
|
458
|
+
results = 1.upto(10).map do |iter|
|
459
|
+
@brt.run_transforms
|
460
|
+
|
461
|
+
Array(target_names).map do |target_name|
|
462
|
+
@brt.targets[target_name].fields[target_field_name].values.uniq
|
463
|
+
end.flatten
|
464
|
+
end.flatten
|
465
|
+
|
466
|
+
results.each do |result_value|
|
467
|
+
expect(result_value).to match(regex_pattern)
|
468
|
+
end
|
469
|
+
|
470
|
+
expect(results.size).to eq results.uniq.size
|
471
|
+
|
472
|
+
end
|
473
|
+
|
474
|
+
Then /^the target field contains a unique value matching the pattern \/(.*)\/$/ do |pattern|
|
475
|
+
@brt.targets.fields.each do |target_field|
|
476
|
+
step "the target field '#{target_field.full_name}' contains a unique value matching the pattern /#{pattern}/"
|
477
|
+
end
|
478
|
+
end
|
479
|
+
|
480
|
+
|
377
481
|
|
378
482
|
### Field presence
|
379
483
|
|
@@ -384,7 +488,7 @@ Then /^only the following fields should be present on the target:$/ do |table|
|
|
384
488
|
end
|
385
489
|
|
386
490
|
@brt.run_transforms
|
387
|
-
expect(@brt.target.data_obj.fields.keys).to match_array @brt.target.fields.
|
491
|
+
expect(@brt.target.data_obj.fields.keys).to match_array @brt.target.fields.field_names
|
388
492
|
end
|
389
493
|
|
390
494
|
### Record-level expectations
|
@@ -0,0 +1,14 @@
|
|
1
|
+
Feature: Test the truncate transformer.
|
2
|
+
|
3
|
+
Background:
|
4
|
+
Given the job is 'Truncate'
|
5
|
+
And the job source 'Source Data'
|
6
|
+
And the job target 'Target Data'
|
7
|
+
|
8
|
+
Scenario: Truncating a field.
|
9
|
+
Given the source 'Source Data'
|
10
|
+
And the target 'Target Data'
|
11
|
+
And the job parameter 'truncate_len' is "5"
|
12
|
+
|
13
|
+
And the source field 'My Field' is set to the value "something"
|
14
|
+
Then the target field 'Truncated Field' is set to the value "somet"
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require_relative '../all_jobs_shared'
|
2
|
+
|
3
|
+
class TruncateJob
|
4
|
+
include AllJobsShared
|
5
|
+
|
6
|
+
define_param :truncate_len, 5
|
7
|
+
define_source :source_data, Remi::DataSource::DataFrame,
|
8
|
+
fields: {
|
9
|
+
:my_field => {}
|
10
|
+
}
|
11
|
+
define_target :target_data, Remi::DataTarget::DataFrame
|
12
|
+
|
13
|
+
define_transform :main, sources: :source_data, targets: :target_data do
|
14
|
+
Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
|
15
|
+
map source(:my_field) .target(:truncated_field)
|
16
|
+
.transform(Remi::Transform[:truncate].(params[:truncate_len].to_i))
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -188,7 +188,7 @@ module Remi::BusinessRules
|
|
188
188
|
end
|
189
189
|
|
190
190
|
def add_subject(subject_name, subject)
|
191
|
-
@subjects[subject_name] ||= DataSubject.new(subject)
|
191
|
+
@subjects[subject_name] ||= DataSubject.new(subject_name, subject)
|
192
192
|
end
|
193
193
|
|
194
194
|
def add_field(full_field_name)
|
@@ -204,11 +204,17 @@ module Remi::BusinessRules
|
|
204
204
|
end
|
205
205
|
|
206
206
|
def fields
|
207
|
-
|
208
|
-
|
209
|
-
|
207
|
+
Enumerator.new do |enum|
|
208
|
+
@subjects.each do |subject_name, subject|
|
209
|
+
subject.fields.each { |field_name, field| enum << field }
|
210
|
+
end
|
210
211
|
end
|
211
|
-
|
212
|
+
end
|
213
|
+
|
214
|
+
def full_field_names
|
215
|
+
@subjects.map do |subject_name, subject|
|
216
|
+
subject.fields.map { |field_name, field| "#{field.full_name}" }
|
217
|
+
end.flatten
|
212
218
|
end
|
213
219
|
|
214
220
|
def size
|
@@ -222,13 +228,15 @@ module Remi::BusinessRules
|
|
222
228
|
|
223
229
|
|
224
230
|
class DataSubject
|
225
|
-
def initialize(subject)
|
231
|
+
def initialize(name, subject)
|
232
|
+
@name = name
|
226
233
|
@data_obj = subject
|
227
234
|
@fields = DataFieldCollection.new
|
228
235
|
|
229
236
|
stub_data
|
230
237
|
end
|
231
238
|
|
239
|
+
attr_reader :name
|
232
240
|
attr_reader :data_obj
|
233
241
|
|
234
242
|
def add_field(field_name)
|
@@ -353,7 +361,7 @@ module Remi::BusinessRules
|
|
353
361
|
generated_data = generate_values_from_cumulative_dist(@data_obj.df.size, cumulative_dist)
|
354
362
|
|
355
363
|
generated_data.each do |field_name, data_array|
|
356
|
-
vector_name = fields[field_name].
|
364
|
+
vector_name = fields[field_name].field_name
|
357
365
|
@data_obj.df[vector_name] = Daru::Vector.new(data_array, index: @data_obj.df.index)
|
358
366
|
end
|
359
367
|
end
|
@@ -403,8 +411,13 @@ module Remi::BusinessRules
|
|
403
411
|
@fields.values.map(&:name)
|
404
412
|
end
|
405
413
|
|
414
|
+
def field_names
|
415
|
+
@fields.values.map(&:field_name)
|
416
|
+
end
|
417
|
+
|
406
418
|
def add_field(subject, field_name)
|
407
|
-
|
419
|
+
raise "Attempting to add a field with the same name but different subject - #{subject.name}: #{field_name}" if @fields.include?(field_name) && @fields[field_name].subject.name != subject.name
|
420
|
+
@fields[field_name] = DataField.new(subject, field_name) unless @fields.include? field_name
|
408
421
|
end
|
409
422
|
|
410
423
|
def only
|
@@ -420,21 +433,26 @@ module Remi::BusinessRules
|
|
420
433
|
|
421
434
|
|
422
435
|
class DataField
|
423
|
-
def initialize(subject,
|
436
|
+
def initialize(subject, name)
|
424
437
|
@subject = subject
|
425
|
-
@
|
438
|
+
@name = name
|
439
|
+
@field_name = name.symbolize(subject.data_obj.field_symbolizer)
|
426
440
|
end
|
427
441
|
|
428
|
-
|
429
|
-
|
442
|
+
attr_reader :name
|
443
|
+
attr_reader :field_name
|
444
|
+
attr_reader :subject
|
445
|
+
|
446
|
+
def full_name
|
447
|
+
"#{@subject.name}: #{@name}"
|
430
448
|
end
|
431
449
|
|
432
450
|
def metadata
|
433
|
-
@subject.fields[
|
451
|
+
@subject.data_obj.fields[@field_name]
|
434
452
|
end
|
435
453
|
|
436
454
|
def vector
|
437
|
-
@subject.df[@field_name]
|
455
|
+
@subject.data_obj.df[@field_name]
|
438
456
|
end
|
439
457
|
|
440
458
|
def value
|
data/lib/remi/transform.rb
CHANGED
@@ -38,6 +38,12 @@ module Remi
|
|
38
38
|
end
|
39
39
|
end
|
40
40
|
|
41
|
+
def truncate(len)
|
42
|
+
memoize_as_lambda(__method__, len) do |(mlen), larg|
|
43
|
+
larg.slice(0,len)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
41
47
|
def concatenate(delimiter="")
|
42
48
|
memoize_as_lambda(__method__, delimiter) do |(mdelimiter), *largs|
|
43
49
|
Array(largs).join(mdelimiter)
|
data/lib/remi/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: remi
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.18
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sterling Paramore
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-03-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: daru
|
@@ -238,6 +238,7 @@ files:
|
|
238
238
|
- features/transforms/nvl.feature
|
239
239
|
- features/transforms/parse_date.feature
|
240
240
|
- features/transforms/prefix.feature
|
241
|
+
- features/transforms/truncate.feature
|
241
242
|
- jobs/aggregate_job.rb
|
242
243
|
- jobs/all_jobs_shared.rb
|
243
244
|
- jobs/copy_source_job.rb
|
@@ -250,6 +251,7 @@ files:
|
|
250
251
|
- jobs/transforms/parse_date_job.rb
|
251
252
|
- jobs/transforms/prefix_job.rb
|
252
253
|
- jobs/transforms/transform_jobs.rb
|
254
|
+
- jobs/transforms/truncate_job.rb
|
253
255
|
- lib/remi.rb
|
254
256
|
- lib/remi/cli.rb
|
255
257
|
- lib/remi/cucumber.rb
|
@@ -318,3 +320,4 @@ test_files:
|
|
318
320
|
- features/transforms/nvl.feature
|
319
321
|
- features/transforms/parse_date.feature
|
320
322
|
- features/transforms/prefix.feature
|
323
|
+
- features/transforms/truncate.feature
|