remi 0.2.17 → 0.2.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +2 -2
- data/features/formulas.feature +2 -1
- data/features/sample_job.feature +4 -4
- data/features/step_definitions/remi_step.rb +175 -71
- data/features/transforms/truncate.feature +14 -0
- data/jobs/transforms/truncate_job.rb +19 -0
- data/lib/remi/cucumber/business_rules.rb +32 -14
- data/lib/remi/transform.rb +6 -0
- data/lib/remi/version.rb +1 -1
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7c112dfed8a5515c7c839e41c038181c79364504
|
4
|
+
data.tar.gz: a7185273e9bb567e70a83bf09a8670cf85d4d223
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 81645e029daa594ec66f7b109443f1337a1e2af0c5d85daf0f54d6e3f6254ad4c02ba637b31b96eb487aa2f3c1a8df535b90f7f67b29b2f61eb291d8bc052443
|
7
|
+
data.tar.gz: 6a700425dc41377ffa5e56aa34a46eb4c7d9d9f9e9d37c48c631379f0cf3336e8fab1188a43d7b908b39dbbfa154176bc0473427932c92b1823795e4504897aa
|
data/Gemfile.lock
CHANGED
data/features/formulas.feature
CHANGED
@@ -72,4 +72,5 @@ Feature: This tests the creation of example records.
|
|
72
72
|
And the source field 'Some Date'
|
73
73
|
And the source field 'Some String'
|
74
74
|
And the target field 'Combination'
|
75
|
-
Then the target field is a concatenation of 'Some Date' and 'Some String', delimited by "-"
|
75
|
+
Then the target field 'Combination' is a concatenation of the source fields 'Some Date' and 'Some String', delimited by "-"
|
76
|
+
Then the target field 'Combination' is a concatenation of the source fields, delimited by "-"
|
data/features/sample_job.feature
CHANGED
@@ -195,12 +195,12 @@ Feature: This is a sample feature file.
|
|
195
195
|
Then the target field '<Target Field>' is populated from the source field using the format "<Target Format>"
|
196
196
|
|
197
197
|
When the source field is blank
|
198
|
-
Then the target field '<Target Field>' is populated with "<If Blank>"
|
198
|
+
Then the target field '<Target Field>' is populated with "<If Blank>"
|
199
199
|
|
200
200
|
Examples:
|
201
|
-
| Source Field | Source Format | Target Field | Target Format | If Blank
|
202
|
-
| Applied Date | %m/%d/%Y | Applied_Date__c | %Y-%m-%d | *Today
|
203
|
-
| Birthdate | %m/%d/%Y | Birthdate | %Y-%m-%d |
|
201
|
+
| Source Field | Source Format | Target Field | Target Format | If Blank |
|
202
|
+
| Applied Date | %m/%d/%Y | Applied_Date__c | %Y-%m-%d | *Today: %Y-%m-%d* |
|
203
|
+
| Birthdate | %m/%d/%Y | Birthdate | %Y-%m-%d | |
|
204
204
|
|
205
205
|
|
206
206
|
Scenario: Populating School__c
|
@@ -106,7 +106,7 @@ Given /^the (source|target) file contains at least the following headers in no p
|
|
106
106
|
field = row.first
|
107
107
|
step "the #{st} field '#{field}'"
|
108
108
|
end
|
109
|
-
expect(@brt.send(st.to_sym).data_obj.fields.keys).to include(*@brt.send(st.to_sym).fields.
|
109
|
+
expect(@brt.send(st.to_sym).data_obj.fields.keys).to include(*@brt.send(st.to_sym).fields.field_names)
|
110
110
|
end
|
111
111
|
|
112
112
|
Given /^the (source|target) file contains all of the following headers in this order:$/ do |st, table|
|
@@ -116,7 +116,7 @@ Given /^the (source|target) file contains all of the following headers in this o
|
|
116
116
|
end
|
117
117
|
|
118
118
|
@brt.run_transforms if st == 'target'
|
119
|
-
expect(@brt.send(st.to_sym).data_obj.fields.keys).to eq @brt.send(st.to_sym).fields.
|
119
|
+
expect(@brt.send(st.to_sym).data_obj.fields.keys).to eq @brt.send(st.to_sym).fields.field_names
|
120
120
|
end
|
121
121
|
|
122
122
|
|
@@ -130,29 +130,31 @@ Given /^the source field '([^']+)'$/ do |source_field_name|
|
|
130
130
|
@brt.sources.add_field(source_field_name)
|
131
131
|
end
|
132
132
|
|
133
|
-
Given /^the source field has the value "([^"]*)"$/ do |
|
134
|
-
|
133
|
+
Given /^the source field '([^']+)' (?:has|is set to) the value "([^"]*)"$/ do |source_field, value|
|
134
|
+
step "the source field '#{source_field}'"
|
135
|
+
|
136
|
+
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
137
|
+
@brt.sources[source_name].fields[source_field_name].value = Remi::BusinessRules::ParseFormula.parse(value)
|
135
138
|
end
|
136
139
|
|
137
|
-
|
138
|
-
@brt.
|
140
|
+
Given /^the source field (?:has|is set to) the value "([^"]*)"$/ do |value|
|
141
|
+
@brt.sources.fields.each do |field|
|
142
|
+
step "the source field '#{field.full_name}' is set to the value \"#{value}\""
|
143
|
+
end
|
139
144
|
end
|
140
145
|
|
141
|
-
When /^the source field '(
|
146
|
+
When /^the source field '([^']+)' (?:has an empty value|is blank)$/ do |source_field|
|
142
147
|
step "the source field '#{source_field}'"
|
148
|
+
|
143
149
|
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
144
|
-
@brt.sources[source_name].fields[source_field_name].value = ''
|
145
|
-
end
|
146
150
|
|
147
|
-
|
148
|
-
step "the source field '#{source_field}'"
|
149
|
-
@brt.source.fields[source_field].value = Remi::BusinessRules::ParseFormula.parse(value)
|
151
|
+
@brt.sources[source_name].fields[source_field_name].value = ''
|
150
152
|
end
|
151
153
|
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
154
|
+
When /^the source field (?:has an empty value|is blank)$/ do
|
155
|
+
@brt.sources.fields.each do |field|
|
156
|
+
step "the source field '#{field.full_name}' is blank"
|
157
|
+
end
|
156
158
|
end
|
157
159
|
|
158
160
|
Given /^the source field '(.+:.+)' (?:has|is set to) the value in the source field '(.+:.+)'$/ do |source_field, other_source_field|
|
@@ -178,8 +180,17 @@ Given /^the source data are tied through the fields '(.+:.+)' and '(.+:.+)'$/ do
|
|
178
180
|
step "the source field '#{other_source_field}' is set to the value in the source field '#{source_field}'"
|
179
181
|
end
|
180
182
|
|
183
|
+
Given /^the source field '([^']+)' is parsed with the date format "([^"]*)"$/ do |source_field, date_format|
|
184
|
+
step "the source field '#{source_field}'"
|
185
|
+
|
186
|
+
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
187
|
+
expect(@brt.sources[source_name].fields[source_field_name].metadata[:format]).to eq date_format
|
188
|
+
end
|
189
|
+
|
181
190
|
Given /^the source field is parsed with the date format "([^"]*)"$/ do |date_format|
|
182
|
-
|
191
|
+
@brt.sources.fields.each do |field|
|
192
|
+
step "the source field '#{field.full_name}' is parsed with the date format \"#{date_format}\""
|
193
|
+
end
|
183
194
|
end
|
184
195
|
|
185
196
|
Given /^the source field is a valid email address$/ do
|
@@ -200,55 +211,66 @@ Given /^the target field '([^']+)'$/ do |arg|
|
|
200
211
|
@brt.targets.add_field(arg)
|
201
212
|
end
|
202
213
|
|
203
|
-
Then /^the target field '(
|
204
|
-
step "the target field '#{arg}'"
|
205
|
-
step "the target field is copied from the source field"
|
206
|
-
end
|
207
|
-
|
208
|
-
Then /^the target field is copied from the source field$/ do
|
209
|
-
@brt.run_transforms
|
210
|
-
expect(@brt.target.field.value).to eq (@brt.source.field.value)
|
211
|
-
end
|
212
|
-
|
213
|
-
Then /^the target field '(.+)' is copied from the source field '(.+:.+)'$/ do |target_field, source_field|
|
214
|
+
Then /^the target field '([^']+)' is copied from the source field '([^']+)'$/ do |target_field, source_field|
|
214
215
|
step "the target field '#{target_field}'"
|
215
216
|
step "the source field '#{source_field}'"
|
216
217
|
|
217
218
|
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
219
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
218
220
|
|
219
221
|
@brt.run_transforms
|
220
|
-
|
222
|
+
Array(target_names).each do |target_name|
|
223
|
+
expect(@brt.targets[target_name].fields[target_field_name].value).to eq (@brt.sources[source_name].fields[source_field_name].value)
|
224
|
+
end
|
221
225
|
end
|
222
226
|
|
223
|
-
Then /^the target field '(
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
@brt.run_transforms
|
228
|
-
expect(@brt.target.field.value).to eq (@brt.source.fields[source_field].value)
|
227
|
+
Then /^the target field '([^']+)' is copied from the source field$/ do |target_field|
|
228
|
+
@brt.sources.fields.each do |source_field|
|
229
|
+
step "the target field '#{target_field}' is copied from the source field '#{source_field.full_name}'"
|
230
|
+
end
|
229
231
|
end
|
230
232
|
|
231
|
-
Then /^the target field is
|
232
|
-
@brt.
|
233
|
-
|
233
|
+
Then /^the target field is copied from the source field$/ do
|
234
|
+
@brt.targets.fields.each do |target_field|
|
235
|
+
@brt.sources.fields.each do |source_field|
|
236
|
+
step "the target field '#{target_field.full_name}' is copied from the source field '#{source_field.full_name}'"
|
237
|
+
end
|
238
|
+
end
|
234
239
|
end
|
235
240
|
|
236
|
-
Then /^the target field '(
|
241
|
+
Then /^the target field '([^']+)' is (?:set to the value|populated with) "([^"]*)"$/ do |target_field, value|
|
237
242
|
expect_cucumber {
|
243
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
244
|
+
|
238
245
|
expect {
|
239
|
-
|
240
|
-
|
246
|
+
Array(target_names).each do |target_name|
|
247
|
+
@brt.targets[target_name].add_field(target_field_name)
|
248
|
+
end
|
249
|
+
@brt.run_transforms
|
241
250
|
}.not_to raise_error
|
242
|
-
|
251
|
+
Array(target_names).each do |target_name|
|
252
|
+
expect(@brt.targets[target_name].fields[target_field_name].values.uniq).to eq [Remi::BusinessRules::ParseFormula.parse(value)]
|
253
|
+
end
|
243
254
|
}
|
244
255
|
end
|
245
256
|
|
246
|
-
Then /^the target field
|
257
|
+
Then /^the target field is (?:set to the value|populated with) "([^"]*)"$/ do |value|
|
258
|
+
@brt.targets.fields.each do |field|
|
259
|
+
step "the target field '#{field.full_name}' is populated with \"#{value}\""
|
260
|
+
end
|
261
|
+
end
|
262
|
+
|
263
|
+
|
264
|
+
Then /^the target field '(.+)' has a value in the list "([^"]*)"$/ do |target_field, list|
|
247
265
|
step "the target field '#{target_field}'"
|
248
266
|
|
267
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
268
|
+
|
249
269
|
list_array = list.split(',').map(&:strip)
|
250
270
|
@brt.run_transforms
|
251
|
-
|
271
|
+
Array(target_names).each do |target_name|
|
272
|
+
expect(@brt.targets[target_name].fields[target_field].values.uniq & list_array).to include(*@brt.targets[target_name].fields[target_field].values.uniq)
|
273
|
+
end
|
252
274
|
end
|
253
275
|
|
254
276
|
|
@@ -284,31 +306,82 @@ end
|
|
284
306
|
|
285
307
|
### Transforms
|
286
308
|
|
309
|
+
Then /^the target field '([^']+)' is a concatenation of the source fields '(.+)', delimited by "([^"]*)"$/ do |target_field, source_field_list, delimiter|
|
310
|
+
source_fields = "'#{source_field_list}'".gsub(' and ', ', ').split(',').map do |field_with_quotes|
|
311
|
+
full_field_name = field_with_quotes.match(/'(.+)'/)[1]
|
312
|
+
|
313
|
+
source_name, field_name = @brt.sources.parse_full_field(full_field_name)
|
314
|
+
{ full_field_name: full_field_name, source: source_name, field: field_name }
|
315
|
+
end
|
316
|
+
|
317
|
+
concatenated_source = source_fields.map do |field|
|
318
|
+
step "the source field '#{field[:full_field_name]}'"
|
319
|
+
@brt.sources[field[:source]].fields[field[:field]].values.uniq
|
320
|
+
end.join(delimiter)
|
321
|
+
|
322
|
+
step "the target field '#{target_field}'"
|
323
|
+
|
324
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
325
|
+
|
326
|
+
@brt.run_transforms
|
327
|
+
Array(target_names).each do |target_name|
|
328
|
+
expect(@brt.targets[target_name].fields[target_field_name].values.uniq).to eq Array(concatenated_source)
|
329
|
+
end
|
330
|
+
end
|
331
|
+
|
332
|
+
Then /^the target field '([^']+)' is a concatenation of the source fields, delimited by "([^"]*)"$/ do |target_field, delimiter|
|
333
|
+
source_field_list = @brt.sources.fields.map do |field|
|
334
|
+
"'#{field.full_name}'"
|
335
|
+
end.join(',')
|
336
|
+
|
337
|
+
step "the target field '#{target_field}' is a concatenation of the source fields #{source_field_list}, delimited by \"#{delimiter}\""
|
338
|
+
end
|
339
|
+
|
287
340
|
Then /^the target field is a concatenation of the source fields, delimited by "([^"]*)"$/ do |delimiter|
|
288
|
-
|
289
|
-
|
341
|
+
@brt.targets.fields.each do |target_field|
|
342
|
+
step "the target field '#{target_field.full_name}' is a concatenation of the source fields, delimited by \"#{delimiter}\""
|
290
343
|
end
|
344
|
+
end
|
345
|
+
|
291
346
|
|
347
|
+
Then /^the target field '([^']+)' is a concatenation of "([^"]*)" and '(.+)', delimited by "([^"]*)"$/ do |target_field, constant, source_field, delimiter|
|
348
|
+
step "the target field '#{target_field}'"
|
349
|
+
|
350
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
351
|
+
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
352
|
+
|
353
|
+
expected_value = [constant, @brt.sources[source_name].fields[source_field_name].value].join(delimiter)
|
292
354
|
@brt.run_transforms
|
293
|
-
|
355
|
+
|
356
|
+
Array(target_names).each do |target_name|
|
357
|
+
expect(@brt.targets[target_name].fields[target_field_name].values.uniq).to eq Array(expected_value)
|
358
|
+
end
|
294
359
|
end
|
295
360
|
|
296
|
-
Then /^the target field is a concatenation of '(.+)' and
|
297
|
-
|
361
|
+
Then /^the target field '([^']+)' is a concatenation of '(.+)' and "([^"]*)", delimited by "([^"]*)"$/ do |target_field, source_field, constant, delimiter|
|
362
|
+
step "the target field '#{target_field}'"
|
363
|
+
|
364
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
365
|
+
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
366
|
+
|
367
|
+
expected_value = [@brt.sources[source_name].fields[source_field_name].value, constant].join(delimiter)
|
298
368
|
@brt.run_transforms
|
299
|
-
|
369
|
+
|
370
|
+
Array(target_names).each do |target_name|
|
371
|
+
expect(@brt.targets[target_name].fields[target_field_name].values.uniq).to eq Array(expected_value)
|
372
|
+
end
|
300
373
|
end
|
301
374
|
|
302
375
|
Then /^the target field is a concatenation of "([^"]*)" and '(.+)', delimited by "([^"]*)"$/ do |constant, source_field, delimiter|
|
303
|
-
|
304
|
-
|
305
|
-
|
376
|
+
@brt.targets.fields.each do |target_field|
|
377
|
+
step "the target field '#{target_field.full_name}' is a concatenation of \"#{constant}\" and '#{source_field}', delimited by \"#{delimiter}\""
|
378
|
+
end
|
306
379
|
end
|
307
380
|
|
308
381
|
Then /^the target field is a concatenation of '(.+)' and "([^"]*)", delimited by "([^"]*)"$/ do |source_field, constant, delimiter|
|
309
|
-
|
310
|
-
|
311
|
-
|
382
|
+
@brt.targets.fields.each do |target_field|
|
383
|
+
step "the target field '#{target_field.full_name}' is a concatenation of '#{source_field}' and \"#{constant}\", delimited by \"#{delimiter}\""
|
384
|
+
end
|
312
385
|
end
|
313
386
|
|
314
387
|
Then /^the source field is prefixed with "([^"]*)" and loaded into the target field$/ do |prefix|
|
@@ -317,25 +390,27 @@ Then /^the source field is prefixed with "([^"]*)" and loaded into the target fi
|
|
317
390
|
expect(@brt.target.field.value).to eq prefixed_source
|
318
391
|
end
|
319
392
|
|
320
|
-
Then /^the target field '(
|
321
|
-
|
393
|
+
Then /^the target field '([^']+)' is populated from the source field '([^']+)' using the format "([^"]*)"$/ do |target_field, source_field, target_format|
|
394
|
+
step "the source field '#{source_field}'"
|
395
|
+
step "the target field '#{target_field}'"
|
396
|
+
|
397
|
+
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
398
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
399
|
+
|
400
|
+
source_format = @brt.sources[source_name].fields[source_field_name].metadata[:format]
|
322
401
|
source_reformatted = Remi::Transform[:format_date].(from_fmt: source_format, to_fmt: target_format)
|
323
|
-
.call(@brt.
|
402
|
+
.call(@brt.sources[source_name].fields[source_field_name].value)
|
324
403
|
|
325
|
-
step "the target field '#{target_field}'"
|
326
404
|
@brt.run_transforms
|
327
|
-
|
405
|
+
target_names.each do |target_name|
|
406
|
+
expect(@brt.targets[target_name].fields[target_field_name].value).to eq source_reformatted
|
407
|
+
end
|
328
408
|
end
|
329
409
|
|
330
|
-
Then /^the target field '(
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
.call(target_value_source_format)
|
335
|
-
|
336
|
-
step "the target field '#{target_field}'"
|
337
|
-
@brt.run_transforms
|
338
|
-
expect(@brt.target.field.value).to eq target_reformatted
|
410
|
+
Then /^the target field '([^']+)' is populated from the source field using the format "([^"]*)"$/ do |target_field, target_format|
|
411
|
+
@brt.sources.fields.each do |source_field|
|
412
|
+
step "the target field '#{target_field}' is populated from the source field '#{source_field.full_name}' using the format \"#{target_format}\""
|
413
|
+
end
|
339
414
|
end
|
340
415
|
|
341
416
|
Then /^the target field '(.+)' is the first non-blank value from source fields '(.+)'$/ do |target_field_name, source_field_list|
|
@@ -374,6 +449,35 @@ Then /^the target field is copied from the source field, but commas have been re
|
|
374
449
|
expect(@brt.target.field.value).to eq source_field_value.gsub(/,/, '.')
|
375
450
|
end
|
376
451
|
|
452
|
+
Then /^the target field '([^']+)' contains a unique value matching the pattern \/(.*)\/$/ do |target_field, pattern|
|
453
|
+
step "the target field '#{target_field}'"
|
454
|
+
|
455
|
+
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
456
|
+
regex_pattern = Regexp.new(pattern)
|
457
|
+
|
458
|
+
results = 1.upto(10).map do |iter|
|
459
|
+
@brt.run_transforms
|
460
|
+
|
461
|
+
Array(target_names).map do |target_name|
|
462
|
+
@brt.targets[target_name].fields[target_field_name].values.uniq
|
463
|
+
end.flatten
|
464
|
+
end.flatten
|
465
|
+
|
466
|
+
results.each do |result_value|
|
467
|
+
expect(result_value).to match(regex_pattern)
|
468
|
+
end
|
469
|
+
|
470
|
+
expect(results.size).to eq results.uniq.size
|
471
|
+
|
472
|
+
end
|
473
|
+
|
474
|
+
Then /^the target field contains a unique value matching the pattern \/(.*)\/$/ do |pattern|
|
475
|
+
@brt.targets.fields.each do |target_field|
|
476
|
+
step "the target field '#{target_field.full_name}' contains a unique value matching the pattern /#{pattern}/"
|
477
|
+
end
|
478
|
+
end
|
479
|
+
|
480
|
+
|
377
481
|
|
378
482
|
### Field presence
|
379
483
|
|
@@ -384,7 +488,7 @@ Then /^only the following fields should be present on the target:$/ do |table|
|
|
384
488
|
end
|
385
489
|
|
386
490
|
@brt.run_transforms
|
387
|
-
expect(@brt.target.data_obj.fields.keys).to match_array @brt.target.fields.
|
491
|
+
expect(@brt.target.data_obj.fields.keys).to match_array @brt.target.fields.field_names
|
388
492
|
end
|
389
493
|
|
390
494
|
### Record-level expectations
|
@@ -0,0 +1,14 @@
|
|
1
|
+
Feature: Test the truncate transformer.
|
2
|
+
|
3
|
+
Background:
|
4
|
+
Given the job is 'Truncate'
|
5
|
+
And the job source 'Source Data'
|
6
|
+
And the job target 'Target Data'
|
7
|
+
|
8
|
+
Scenario: Truncating a field.
|
9
|
+
Given the source 'Source Data'
|
10
|
+
And the target 'Target Data'
|
11
|
+
And the job parameter 'truncate_len' is "5"
|
12
|
+
|
13
|
+
And the source field 'My Field' is set to the value "something"
|
14
|
+
Then the target field 'Truncated Field' is set to the value "somet"
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require_relative '../all_jobs_shared'
|
2
|
+
|
3
|
+
class TruncateJob
|
4
|
+
include AllJobsShared
|
5
|
+
|
6
|
+
define_param :truncate_len, 5
|
7
|
+
define_source :source_data, Remi::DataSource::DataFrame,
|
8
|
+
fields: {
|
9
|
+
:my_field => {}
|
10
|
+
}
|
11
|
+
define_target :target_data, Remi::DataTarget::DataFrame
|
12
|
+
|
13
|
+
define_transform :main, sources: :source_data, targets: :target_data do
|
14
|
+
Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
|
15
|
+
map source(:my_field) .target(:truncated_field)
|
16
|
+
.transform(Remi::Transform[:truncate].(params[:truncate_len].to_i))
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -188,7 +188,7 @@ module Remi::BusinessRules
|
|
188
188
|
end
|
189
189
|
|
190
190
|
def add_subject(subject_name, subject)
|
191
|
-
@subjects[subject_name] ||= DataSubject.new(subject)
|
191
|
+
@subjects[subject_name] ||= DataSubject.new(subject_name, subject)
|
192
192
|
end
|
193
193
|
|
194
194
|
def add_field(full_field_name)
|
@@ -204,11 +204,17 @@ module Remi::BusinessRules
|
|
204
204
|
end
|
205
205
|
|
206
206
|
def fields
|
207
|
-
|
208
|
-
|
209
|
-
|
207
|
+
Enumerator.new do |enum|
|
208
|
+
@subjects.each do |subject_name, subject|
|
209
|
+
subject.fields.each { |field_name, field| enum << field }
|
210
|
+
end
|
210
211
|
end
|
211
|
-
|
212
|
+
end
|
213
|
+
|
214
|
+
def full_field_names
|
215
|
+
@subjects.map do |subject_name, subject|
|
216
|
+
subject.fields.map { |field_name, field| "#{field.full_name}" }
|
217
|
+
end.flatten
|
212
218
|
end
|
213
219
|
|
214
220
|
def size
|
@@ -222,13 +228,15 @@ module Remi::BusinessRules
|
|
222
228
|
|
223
229
|
|
224
230
|
class DataSubject
|
225
|
-
def initialize(subject)
|
231
|
+
def initialize(name, subject)
|
232
|
+
@name = name
|
226
233
|
@data_obj = subject
|
227
234
|
@fields = DataFieldCollection.new
|
228
235
|
|
229
236
|
stub_data
|
230
237
|
end
|
231
238
|
|
239
|
+
attr_reader :name
|
232
240
|
attr_reader :data_obj
|
233
241
|
|
234
242
|
def add_field(field_name)
|
@@ -353,7 +361,7 @@ module Remi::BusinessRules
|
|
353
361
|
generated_data = generate_values_from_cumulative_dist(@data_obj.df.size, cumulative_dist)
|
354
362
|
|
355
363
|
generated_data.each do |field_name, data_array|
|
356
|
-
vector_name = fields[field_name].
|
364
|
+
vector_name = fields[field_name].field_name
|
357
365
|
@data_obj.df[vector_name] = Daru::Vector.new(data_array, index: @data_obj.df.index)
|
358
366
|
end
|
359
367
|
end
|
@@ -403,8 +411,13 @@ module Remi::BusinessRules
|
|
403
411
|
@fields.values.map(&:name)
|
404
412
|
end
|
405
413
|
|
414
|
+
def field_names
|
415
|
+
@fields.values.map(&:field_name)
|
416
|
+
end
|
417
|
+
|
406
418
|
def add_field(subject, field_name)
|
407
|
-
|
419
|
+
raise "Attempting to add a field with the same name but different subject - #{subject.name}: #{field_name}" if @fields.include?(field_name) && @fields[field_name].subject.name != subject.name
|
420
|
+
@fields[field_name] = DataField.new(subject, field_name) unless @fields.include? field_name
|
408
421
|
end
|
409
422
|
|
410
423
|
def only
|
@@ -420,21 +433,26 @@ module Remi::BusinessRules
|
|
420
433
|
|
421
434
|
|
422
435
|
class DataField
|
423
|
-
def initialize(subject,
|
436
|
+
def initialize(subject, name)
|
424
437
|
@subject = subject
|
425
|
-
@
|
438
|
+
@name = name
|
439
|
+
@field_name = name.symbolize(subject.data_obj.field_symbolizer)
|
426
440
|
end
|
427
441
|
|
428
|
-
|
429
|
-
|
442
|
+
attr_reader :name
|
443
|
+
attr_reader :field_name
|
444
|
+
attr_reader :subject
|
445
|
+
|
446
|
+
def full_name
|
447
|
+
"#{@subject.name}: #{@name}"
|
430
448
|
end
|
431
449
|
|
432
450
|
def metadata
|
433
|
-
@subject.fields[
|
451
|
+
@subject.data_obj.fields[@field_name]
|
434
452
|
end
|
435
453
|
|
436
454
|
def vector
|
437
|
-
@subject.df[@field_name]
|
455
|
+
@subject.data_obj.df[@field_name]
|
438
456
|
end
|
439
457
|
|
440
458
|
def value
|
data/lib/remi/transform.rb
CHANGED
@@ -38,6 +38,12 @@ module Remi
|
|
38
38
|
end
|
39
39
|
end
|
40
40
|
|
41
|
+
def truncate(len)
|
42
|
+
memoize_as_lambda(__method__, len) do |(mlen), larg|
|
43
|
+
larg.slice(0,len)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
41
47
|
def concatenate(delimiter="")
|
42
48
|
memoize_as_lambda(__method__, delimiter) do |(mdelimiter), *largs|
|
43
49
|
Array(largs).join(mdelimiter)
|
data/lib/remi/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: remi
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.18
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sterling Paramore
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-03-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: daru
|
@@ -238,6 +238,7 @@ files:
|
|
238
238
|
- features/transforms/nvl.feature
|
239
239
|
- features/transforms/parse_date.feature
|
240
240
|
- features/transforms/prefix.feature
|
241
|
+
- features/transforms/truncate.feature
|
241
242
|
- jobs/aggregate_job.rb
|
242
243
|
- jobs/all_jobs_shared.rb
|
243
244
|
- jobs/copy_source_job.rb
|
@@ -250,6 +251,7 @@ files:
|
|
250
251
|
- jobs/transforms/parse_date_job.rb
|
251
252
|
- jobs/transforms/prefix_job.rb
|
252
253
|
- jobs/transforms/transform_jobs.rb
|
254
|
+
- jobs/transforms/truncate_job.rb
|
253
255
|
- lib/remi.rb
|
254
256
|
- lib/remi/cli.rb
|
255
257
|
- lib/remi/cucumber.rb
|
@@ -318,3 +320,4 @@ test_files:
|
|
318
320
|
- features/transforms/nvl.feature
|
319
321
|
- features/transforms/parse_date.feature
|
320
322
|
- features/transforms/prefix.feature
|
323
|
+
- features/transforms/truncate.feature
|