remi 0.0.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/.bundle/config +2 -0
  3. data/.gitignore +3 -2
  4. data/.rspec +2 -0
  5. data/.ruby-version +1 -0
  6. data/Gemfile +4 -0
  7. data/Gemfile.lock +123 -0
  8. data/LICENSE.txt +21 -0
  9. data/README.md +94 -3
  10. data/bin/remi +8 -0
  11. data/doc/install-rbenv-os_x.md +47 -0
  12. data/lib/remi.rb +56 -9
  13. data/lib/remi/cli.rb +56 -0
  14. data/lib/remi/core/daru.rb +28 -0
  15. data/lib/remi/core/refinements.rb +21 -0
  16. data/lib/remi/core/string.rb +8 -0
  17. data/lib/remi/cucumber.rb +7 -0
  18. data/lib/remi/cucumber/business_rules.rb +504 -0
  19. data/lib/remi/cucumber/data_source.rb +63 -0
  20. data/lib/remi/data_source.rb +13 -0
  21. data/lib/remi/data_source/csv_file.rb +79 -0
  22. data/lib/remi/data_source/data_frame.rb +10 -0
  23. data/lib/remi/data_source/postgres.rb +58 -0
  24. data/lib/remi/data_source/salesforce.rb +78 -0
  25. data/lib/remi/data_subject.rb +25 -0
  26. data/lib/remi/data_target.rb +15 -0
  27. data/lib/remi/data_target/csv_file.rb +49 -0
  28. data/lib/remi/data_target/data_frame.rb +14 -0
  29. data/lib/remi/data_target/salesforce.rb +49 -0
  30. data/lib/remi/extractor/sftp_file.rb +84 -0
  31. data/lib/remi/field_symbolizers.rb +17 -0
  32. data/lib/remi/job.rb +200 -0
  33. data/lib/remi/lookup/regex_sieve.rb +55 -0
  34. data/lib/remi/project/features/examples.feature +24 -0
  35. data/lib/remi/project/features/formulas.feature +64 -0
  36. data/lib/remi/project/features/sample_job.feature +304 -0
  37. data/lib/remi/project/features/step_definitions/remi_step.rb +310 -0
  38. data/lib/remi/project/features/support/env.rb +10 -0
  39. data/lib/remi/project/features/support/env_app.rb +3 -0
  40. data/lib/remi/project/features/transforms/date_diff.feature +50 -0
  41. data/lib/remi/project/features/transforms/parse_date.feature +34 -0
  42. data/lib/remi/project/features/transforms/prefix.feature +15 -0
  43. data/lib/remi/project/jobs/all_jobs_shared.rb +25 -0
  44. data/lib/remi/project/jobs/copy_source_job.rb +12 -0
  45. data/lib/remi/project/jobs/sample_job.rb +164 -0
  46. data/lib/remi/project/jobs/transforms/date_diff_job.rb +17 -0
  47. data/lib/remi/project/jobs/transforms/parse_date_job.rb +18 -0
  48. data/lib/remi/project/jobs/transforms/prefix_job.rb +16 -0
  49. data/lib/remi/project/jobs/transforms/transform_jobs.rb +3 -0
  50. data/lib/remi/settings.rb +39 -0
  51. data/lib/remi/sf_bulk_helper.rb +265 -0
  52. data/lib/remi/source_to_target_map.rb +93 -0
  53. data/lib/remi/transform.rb +137 -0
  54. data/lib/remi/version.rb +3 -0
  55. data/remi.gemspec +25 -7
  56. data/workbooks/sample_workbook.ipynb +56 -0
  57. data/workbooks/workbook_helper.rb +1 -0
  58. metadata +234 -17
  59. data/lib/noodling.rb +0 -163
  60. data/test/test_NAME.rb +0 -19
@@ -0,0 +1,304 @@
1
+ Feature: This is a sample feature file.
2
+ It demonstrates some of the functionality of using Remi for BRDD.
3
+
4
+ Background:
5
+ Given the job is 'Sample'
6
+ And the job source 'Sample File'
7
+ And the job source 'Existing Contacts'
8
+ And the job target 'Contact Updates'
9
+ And the job target 'Contact Creates'
10
+
11
+ And the following example record called 'existing contact':
12
+ | Id | External_ID__c |
13
+ | 003G0030H1BxH0xIIF | SAMP12345 |
14
+
15
+ And the following example record called 'sample records to load':
16
+ | Program | Student Id |
17
+ | BIO | 12345 |
18
+ | BIO | 0 |
19
+
20
+ And the following example record called 'sample record to update':
21
+ | Program | Student Id |
22
+ | BIO | 12345 |
23
+
24
+ And the following example record called 'sample record to create':
25
+ | Program | Student Id |
26
+ | BIO | 0 |
27
+
28
+ ##### File Specifications #####
29
+
30
+ Scenario: Selecting and downloading the appropriate source files
31
+
32
+ Given the source 'Sample File'
33
+ And files with names matching the pattern /^SampleFile_(\d+)\.txt/
34
+ Then the file with the latest date stamp will be downloaded for processing
35
+
36
+ Given files with names that do not match the pattern /^SampleFile_(\d+)\.txt/
37
+ Then no files will be downloaded for processing
38
+
39
+
40
+ Scenario: In order to be parsed and properly processed, the file must conform
41
+ to expectations about its structure and content.
42
+
43
+ Given the source 'Sample File'
44
+ And the source file is delimited with a comma
45
+ And the source file is encoded using "ISO-8859-1" format
46
+ And the source file uses a double quote to quote embedded delimiters
47
+ And the source file uses a preceeding double quote to escape an embedded quoting character
48
+ And the source file uses windows or unix line endings
49
+ And the source file contains a header row
50
+ And the source file contains at least the following headers in no particular order:
51
+ | header |
52
+ | Student Id |
53
+ | School Id |
54
+ | School Name |
55
+ | Program |
56
+ | Last Name |
57
+ | First Name |
58
+ | Current Email |
59
+ | Mailing Address Line 1 |
60
+ | Mailing Address Line 2 |
61
+ | Mailing City |
62
+ | Mailing State |
63
+ | Mailing Postal Code |
64
+ | Birthdate |
65
+ | Applied Date |
66
+
67
+
68
+ ##### Record Acceptance #####
69
+
70
+ Scenario Outline: 'Sample File' records are rejected for non-matching programs.
71
+ Records are rejected from all targets without error unless the
72
+ 'Program' name is included in the list of acceptable program
73
+ names. The examples below are not exhaustive.
74
+
75
+
76
+ Given the source 'Sample File'
77
+ And the target 'Contact Updates'
78
+ And the target 'Contact Creates'
79
+ And the example 'sample records to load' for 'Sample File'
80
+ And the example 'existing contact' for 'Existing Contacts'
81
+
82
+ When the source field 'Program' has the value "<Program>"
83
+ Then the record should be <Action> without error
84
+
85
+ Examples:
86
+ | Program | Action |
87
+ | BIO | Retained |
88
+ | Fake Biology | Rejected |
89
+ | COMM | Rejected |
90
+ | CHEM | Retained |
91
+
92
+
93
+ ##### Transformations common to both contact updates and creates #####
94
+
95
+ Scenario Outline: Populating Major__c for updates and creates.
96
+ This scenario uses the program name mapping. The examples below
97
+ are not exhaustive, but are intended for demonstration and testing
98
+ of edge cases.
99
+
100
+ Given the source 'Sample File'
101
+ And the target 'Contact Updates'
102
+ And the target 'Contact Creates'
103
+ And the example 'existing contact' for 'Existing Contacts'
104
+ And the example 'sample records to load' for 'Sample File'
105
+
106
+ When the source field 'Program' has the value "<Program>"
107
+ Then the target field 'Major__c' is set to the value "<Major__c>"
108
+
109
+ Examples:
110
+ | Program | Major__c |
111
+ | BIO | Biology |
112
+ | CHEM | Chemistry |
113
+ | Chemistry | Chemistry |
114
+ | Physical Chemistry | Chemistry |
115
+ | Microbiology | Biology |
116
+
117
+
118
+ ##### Transformations for creating contact records #####
119
+
120
+ Scenario: Fields that are present on create
121
+ Given the target 'Contact Creates'
122
+ And the example 'existing contact' for 'Existing Contacts'
123
+ And the example 'sample record to create' for 'Sample File'
124
+
125
+ Then only the following fields should be present on the target:
126
+ | Field Name |
127
+ | External_ID__c |
128
+ | School_ID__c |
129
+ | School_Name__c |
130
+ | School__c |
131
+ | Major__c |
132
+ | FirstName |
133
+ | LastName |
134
+ | Email |
135
+ | MailingStreet |
136
+ | MailingCity |
137
+ | MailingState |
138
+ | MailingPostalCode |
139
+ | Birthdate |
140
+ | Applied_Date__c |
141
+
142
+
143
+ Scenario: Records that are directed to Contact creates
144
+ This example shows how to create a scenario that simultaneously references
145
+ two sources.
146
+
147
+ Given the source 'Sample File'
148
+ And the source 'Existing Contacts'
149
+ And the target 'Contact Creates'
150
+ And the example 'sample record to create' for 'Sample File'
151
+ And the example 'existing contact' for 'Existing Contacts'
152
+
153
+ And the source field 'Sample File: Student Id'
154
+ And the source field 'Existing Contacts: External_ID__c' has the value "some arbitrary value"
155
+ Then a target record is created
156
+
157
+ When the source field 'Existing Contacts: External_ID__c' has the value in the source field 'Sample File: Student Id', prefixed with "SAMP"
158
+ Then a target record is not created
159
+
160
+
161
+ Scenario Outline: Fields that are copied from the 'Sample File' to 'Contact Creates'
162
+ This example shows how a scenario can be used to specify very simple
163
+ field mappings, like a simple copy with missing value handling.
164
+
165
+ Given the source 'Sample File'
166
+ And the target 'Contact Creates'
167
+ And the example 'sample record to create' for 'Sample File'
168
+
169
+ And the source field '<Source Field>'
170
+ Then the target field '<Target Field>' is copied from the source field
171
+
172
+ When the source field is blank
173
+ Then the target field '<Target Field>' is populated with "<If Blank>"
174
+
175
+ Examples:
176
+ | Source Field | Target Field | If Blank |
177
+ | School Id | School_ID__c | |
178
+ | School Name | School_Name__c | |
179
+ | First Name | FirstName | Not Provided |
180
+ | Last Name | LastName | Not Provided |
181
+ | Mailing City | MailingCity | |
182
+ | Mailing State | MailingState | |
183
+ | Mailing Postal Code | MailingPostalCode | |
184
+
185
+
186
+ Scenario Outline: Date fields that are parsed and then copied from the Sample File
187
+ to 'Contact Creates'
188
+
189
+ Given the source 'Sample File'
190
+ And the target 'Contact Creates'
191
+ And the example 'sample record to create' for 'Sample File'
192
+
193
+ And the source field '<Source Field>'
194
+ And the source field is parsed with the date format "<Source Format>"
195
+ Then the target field '<Target Field>' is populated from the source field using the format "<Target Format>"
196
+
197
+ When the source field is blank
198
+ Then the target field '<Target Field>' is populated with "<If Blank>" using the format "<Target Format>"
199
+
200
+ Examples:
201
+ | Source Field | Source Format | Target Field | Target Format | If Blank |
202
+ | Applied Date | %m/%d/%Y | Applied_Date__c | %Y-%m-%d | *Today's Date* |
203
+ | Birthdate | %m/%d/%Y | Birthdate | %Y-%m-%d | |
204
+
205
+
206
+ Scenario: Populating School__c
207
+ This is an example of creating a field that is a delimited
208
+ concatenation of two fields, where we replace blank values with "Unknowns"
209
+
210
+ Given the source 'Sample File'
211
+ And the target 'Contact Creates'
212
+ And the example 'existing contact' for 'Existing Contacts'
213
+ And the example 'sample records to load' for 'Sample File'
214
+
215
+ And the source field 'School Id'
216
+ And the source field 'School Name'
217
+ And the target field 'School__c'
218
+ Then the target field is a concatenation of the source fields, delimited by "-"
219
+
220
+ When the source field 'School Id' is blank
221
+ And the source field 'School Name' has the value "some arbitrary value"
222
+ Then the target field is a concatenation of "Unknown" and 'School Name', delimited by "-"
223
+
224
+ When the source field 'School Id' has the value "some arbitrary value"
225
+ And the source field 'School Name' is blank
226
+ Then the target field is a concatenation of 'School Id' and "Unknown", delimited by "-"
227
+
228
+
229
+ Scenario: Populating Email
230
+ This example shows some pre-cleaning of e-mail addresses (replacing commas with periods)
231
+
232
+ Given the source 'Sample File'
233
+ And the target 'Contact Creates'
234
+ And the example 'sample record to create' for 'Sample File'
235
+
236
+ And the source field 'Current Email'
237
+ And the target field 'Email'
238
+
239
+ When the source field is a valid email address
240
+ Then the target field is copied from the source field
241
+
242
+ When the source field is not a valid email address
243
+ Then the target field is populated with ""
244
+
245
+ When the source field is a valid email address
246
+ And in the source field, periods have been used in place of commas
247
+ Then the target field is copied from the source field, but commas have been replaced by periods
248
+
249
+
250
+ Scenario: Populating External_ID__c
251
+
252
+ Given the source 'Sample File'
253
+ And the target 'Contact Creates'
254
+ And the example 'sample record to create' for 'Sample File'
255
+
256
+ And the source field 'Student Id'
257
+ And the target field 'External_ID__c'
258
+ Then the source field is prefixed with "SAMP" and loaded into the target field
259
+
260
+
261
+ Scenario: Populating MailingStreet
262
+ Given the source 'Sample File'
263
+ And the target 'Contact Creates'
264
+ And the example 'sample record to create' for 'Sample File'
265
+
266
+ And the source field 'Mailing Address Line 1'
267
+ And the source field 'Mailing Address Line 2'
268
+ And the target field 'MailingStreet'
269
+ Then the target field is a concatenation of the source fields, delimited by ", "
270
+
271
+ When the source field 'Mailing Address Line 1' is blank
272
+ Then the target field is populated with ""
273
+
274
+
275
+ ##### Transformations for updating contact records #####
276
+
277
+ Scenario: Fields that are present on update
278
+ The business rules specify that once a student record has been
279
+ created, only updates to the program are accepted (all other
280
+ changes to be done in Salesforce)
281
+
282
+ Given the target 'Contact Updates'
283
+ And the example 'existing contact' for 'Existing Contacts'
284
+ And the example 'sample record to create' for 'Sample File'
285
+
286
+ Then only the following fields should be present on the target:
287
+ | Field Name |
288
+ | Id |
289
+ | Major__c |
290
+
291
+ Scenario: Records that are directed to Contact updates
292
+
293
+ Given the source 'Sample File'
294
+ And the source 'Existing Contacts'
295
+ And the target 'Contact Updates'
296
+ And the example 'sample record to update' for 'Sample File'
297
+ And the example 'existing contact' for 'Existing Contacts'
298
+
299
+ And the source field 'Sample File: Student Id'
300
+ And the source field 'Existing Contacts: External_ID__c' has the value in the source field 'Sample File: Student Id', prefixed with "SAMP"
301
+ Then a target record is created
302
+
303
+ When the source field 'Existing Contacts: External_ID__c' has the value "some arbitrary value"
304
+ Then a target record is not created
@@ -0,0 +1,310 @@
1
+ # Auto-generated from Remi. Editing this file is not recommended. Instead,
2
+ # create another step definition file to contain application-specific steps.
3
+
4
+ ### Job and background setup
5
+
6
+ Given /^the job is '([[:alnum:]\s]+)'$/ do |arg|
7
+ @brt = Remi::BusinessRules::Tester.new(arg)
8
+ end
9
+
10
+ Given /^the job source '([[:alnum:]\s\-_]+)'$/ do |arg|
11
+ @brt.add_job_source arg
12
+ end
13
+
14
+ Given /^the job target '([[:alnum:]\s\-_]+)'$/ do |arg|
15
+ @brt.add_job_target arg
16
+ end
17
+
18
+ Given /^the following example record called '([[:alnum:]\s\-_]+)':$/ do |arg, example_table|
19
+ @brt.add_example arg, example_table
20
+ end
21
+
22
+ Given /^the job parameter '([[:alnum:]\s\-_]+)' is "([^"]*)"$/ do |param, value|
23
+ @brt.set_job_parameter(param, value)
24
+ end
25
+
26
+ ### Setting up example data
27
+
28
+ Given /^the following example record for '([[:alnum:]\s\-_]+)':$/ do |source_name, example_table|
29
+ example_name = source_name
30
+ @brt.add_example example_name, example_table
31
+ @brt.job_sources[source_name].stub_data_with(@brt.examples[example_name])
32
+ end
33
+
34
+ Given /^the example '([[:alnum:]\s\-_]+)' for '([[:alnum:]\s\-_]+)'$/ do |example_name, source_name|
35
+ @brt.job_sources[source_name].stub_data_with(@brt.examples[example_name])
36
+ end
37
+
38
+
39
+ ### Source file processing
40
+
41
+ Given /^files with names matching the pattern \/(.*)\/$/ do |pattern|
42
+ @brt.filestore.pattern(Regexp.new(pattern))
43
+ end
44
+
45
+ Given /^files with names that do not match the pattern \/(.*)\/$/ do |pattern|
46
+ @brt.filestore.anti_pattern(Regexp.new(pattern))
47
+ end
48
+
49
+ Given /^files delivered within the last (\d+) hours$/ do |hours|
50
+ @brt.filestore.delivered_since(Time.now - hours.to_i * 3600)
51
+ end
52
+
53
+ Given /^files were delivered more than (\d+) hours ago$/ do |hours|
54
+ @brt.filestore.delivered_before(Time.now - hours.to_i * 3600)
55
+ end
56
+
57
+ Then /^the file with the latest date stamp will be downloaded for processing$/ do
58
+ @brt.filestore.generate
59
+ @brt.source.mock_extractor(@brt.filestore)
60
+ expect(@brt.source.extract).to match_array Array(@brt.filestore.latest)
61
+ end
62
+
63
+ Then /^no files will be downloaded for processing$/ do
64
+ @brt.filestore.generate
65
+ @brt.source.mock_extractor(@brt.filestore)
66
+ expect { @brt.source.extract }.to raise_error Remi::Extractor::SftpFile::FileNotFoundError
67
+ end
68
+
69
+
70
+ Given /^the source file is delimited with a (\w+)$/ do |delimiter|
71
+ expect(@brt.source.csv_options[:col_sep]).to eq Remi::BusinessRules.csv_opt_map[delimiter]
72
+ end
73
+
74
+ Given /^the source file is encoded using "([^"]+)" format$/ do |encoding|
75
+ expect(@brt.source.csv_options[:encoding].split(':').first).to eq encoding
76
+ end
77
+
78
+ Given /^the source file uses a ([\w ]+) to quote embedded delimiters$/ do |quote_char|
79
+ expect(@brt.source.csv_options[:quote_char]).to eq Remi::BusinessRules.csv_opt_map[quote_char]
80
+ end
81
+
82
+ Given /^the source file uses a preceeding ([\w ]+) to escape an embedded quoting character$/ do |escape_char|
83
+ expect(@brt.source.csv_options[:quote_char]).to eq Remi::BusinessRules.csv_opt_map[escape_char]
84
+ end
85
+
86
+ Given /^the source file uses ([\w ]+) line endings$/ do |line_endings|
87
+ expect(@brt.source.csv_options[:row_sep]).to eq Remi::BusinessRules.csv_opt_map[line_endings]
88
+ end
89
+
90
+ Given /^the source file (contains|does not contain) a header row$/ do |header|
91
+ expect(@brt.source.csv_options[:headers]).to eq (header == 'contains')
92
+ end
93
+
94
+ Given /^the source file contains at least the following headers in no particular order:$/ do |table|
95
+ table.rows.each do |row|
96
+ field = row.first
97
+ step "the source field '#{field}'"
98
+ end
99
+ expect(@brt.source.data_obj.fields.keys).to include(*@brt.source.fields.names)
100
+ end
101
+
102
+
103
+ ### Source
104
+
105
+ Given /^the source '([[:alnum:]\s\-_]+)'$/ do |arg|
106
+ @brt.add_source(arg)
107
+ end
108
+
109
+ Given /^the source field '(.+)'$/ do |arg|
110
+ @brt.sources.add_field(arg)
111
+ end
112
+
113
+ Given /^the source field has the value "([^"]*)"$/ do |arg|
114
+ @brt.source.field.value = Remi::BusinessRules::ParseFormula.parse(arg)
115
+ end
116
+
117
+ When /^the source field (?:has an empty value|is blank)$/ do
118
+ @brt.source.field.value = ''
119
+ end
120
+
121
+ When /^the source field '(.+)' (?:has an empty value|is blank)$/ do |source_field|
122
+ step "the source field '#{source_field}'"
123
+ @brt.source.fields[source_field].value = ''
124
+ end
125
+
126
+ Given /^the source field '([^:]+)' (?:has|is set to) the value "([^"]*)"$/ do |source_field, value|
127
+ step "the source field '#{source_field}'"
128
+ @brt.source.fields[source_field].value = Remi::BusinessRules::ParseFormula.parse(value)
129
+ end
130
+
131
+ Given /^the source field '(.+:.+)' (?:has|is set to) the value "([^"]*)"$/ do |source_field, value|
132
+ step "the source field '#{source_field}'"
133
+ source_name, field_name = *Remi::BusinessRules.parse_full_field(source_field)
134
+ @brt.sources[source_name].fields[field_name].value = Remi::BusinessRules::ParseFormula.parse(value)
135
+ end
136
+
137
+ Given /^the source field '(.+:.+)' (?:has|is set to) the value in the source field '(.+:.+)', prefixed with "([^"]*)"$/ do |source_field, other_source_field, prefix|
138
+ step "the source field '#{source_field}'"
139
+ step "the source field '#{other_source_field}'"
140
+ source_name, field_name = *Remi::BusinessRules.parse_full_field(source_field)
141
+ other_source_name, other_field_name = *Remi::BusinessRules.parse_full_field(other_source_field)
142
+
143
+ prefixed = "#{prefix}#{@brt.sources[other_source_name].fields[other_field_name].value}"
144
+ @brt.sources[source_name].fields[field_name].value = prefixed
145
+ end
146
+
147
+ Given /^the source field is parsed with the date format "([^"]*)"$/ do |date_format|
148
+ expect(@brt.source.field.metadata[:format]).to eq date_format
149
+ end
150
+
151
+ Given /^the source field is a valid email address$/ do
152
+ @brt.source.field.value = 'valid@example.com'
153
+ end
154
+
155
+ Given /^the source field is not a valid email address$/ do
156
+ @brt.source.field.value = 'invalid!example.com'
157
+ end
158
+
159
+ ### Target
160
+
161
+ Given /^the target '([[:alnum:]\s\-_]+)'$/ do |arg|
162
+ @brt.add_target(arg)
163
+ end
164
+
165
+ Given /^the target field '([^']+)'$/ do |arg|
166
+ @brt.targets.add_field(arg)
167
+ end
168
+
169
+ Then /^the target field '(.+)' is copied from the source field$/ do |arg|
170
+ step "the target field '#{arg}'"
171
+ step "the target field is copied from the source field"
172
+ end
173
+
174
+ Then /^the target field is copied from the source field$/ do
175
+ @brt.run_transforms
176
+ expect(@brt.target.field.value).to eq (@brt.source.field.value)
177
+ end
178
+
179
+ Then /^the target field '(.+)' is copied from the source field '(.+:.+)'$/ do |target_field, source_field|
180
+ step "the target field '#{target_field}'"
181
+ step "the source field '#{source_field}'"
182
+
183
+ source_name, source_field_name = *Remi::BusinessRules.parse_full_field(source_field)
184
+
185
+ @brt.run_transforms
186
+ expect(@brt.target.field.value).to eq (@brt.sources[source_name].fields[source_field_name].value)
187
+ end
188
+
189
+ Then /^the target field is (?:set to the value|populated with) "([^"]*)"$/ do |value|
190
+ @brt.run_transforms
191
+ expect(@brt.target.field.value).to eq Remi::BusinessRules::ParseFormula.parse(value)
192
+ end
193
+
194
+ Then /^the target field '(.+)' is (?:set to the value|populated with) "([^"]*)"$/ do |target_field, value|
195
+ @brt.targets.add_field(target_field)
196
+ @brt.run_transforms
197
+ expect(@brt.targets.fields[target_field].values.uniq).to eq [Remi::BusinessRules::ParseFormula.parse(value)]
198
+ end
199
+
200
+ Then /^the target field '(.+)' is the date (.+)$/ do |target_field, date_reference|
201
+ step "the target field '#{target_field}' is set to the value \"*#{date_reference}*\""
202
+ end
203
+
204
+
205
+ ### Transforms
206
+
207
+ Then /^the target field is a concatenation of the source fields, delimited by "([^"]*)"$/ do |delimiter|
208
+ concatenated_source = @brt.sources.fields.values.uniq.map do |row|
209
+ Array(row.join(delimiter))
210
+ end
211
+
212
+ @brt.run_transforms
213
+ expect(@brt.targets.fields.values.uniq).to eq concatenated_source
214
+ end
215
+
216
+ Then /^the target field is a concatenation of "([^"]*)" and '(.+)', delimited by "([^"]*)"$/ do |constant, source_field, delimiter|
217
+ expected_value = [constant, @brt.sources.fields[source_field].value].join(delimiter)
218
+ @brt.run_transforms
219
+ expect(@brt.targets.fields.values.uniq).to eq [[expected_value]]
220
+ end
221
+
222
+ Then /^the target field is a concatenation of '(.+)' and "([^"]*)", delimited by "([^"]*)"$/ do |source_field, constant, delimiter|
223
+ expected_value = [@brt.sources.fields[source_field].value, constant].join(delimiter)
224
+ @brt.run_transforms
225
+ expect(@brt.targets.fields.values.uniq).to eq [[expected_value]]
226
+ end
227
+
228
+ Then /^the source field is prefixed with "([^"]*)" and loaded into the target field$/ do |prefix|
229
+ prefixed_source = "#{prefix}#{@brt.source.field.value}"
230
+ @brt.run_transforms
231
+ expect(@brt.target.field.value).to eq prefixed_source
232
+ end
233
+
234
+ Then /^the target field '(.+)' is populated from the source field using the format "([^"]*)"$/ do |target_field, target_format|
235
+ source_format = @brt.source.field.metadata[:format]
236
+ source_reformatted = Remi::Transform[:format_date].(from_fmt: source_format, to_fmt: target_format)
237
+ .call(@brt.source.field.value)
238
+
239
+ step "the target field '#{target_field}'"
240
+ @brt.run_transforms
241
+ expect(@brt.target.field.value).to eq source_reformatted
242
+ end
243
+
244
+ Then /^the target field '(.+)' is populated with "([^"]*)" using the format "([^"]*)"$/ do |target_field, target_value, target_format|
245
+ source_format = @brt.source.field.metadata[:format]
246
+ target_value_source_format = target_value == "*Today's Date*" ? Date.today.strftime(source_format) : target_value
247
+ target_reformatted = Remi::Transform[:format_date].(from_fmt: source_format, to_fmt: target_format)
248
+ .call(target_value_source_format)
249
+
250
+ step "the target field '#{target_field}'"
251
+ @brt.run_transforms
252
+ expect(@brt.target.field.value).to eq target_reformatted
253
+ end
254
+
255
+
256
+ When /^in the source field, periods have been used in place of commas$/ do
257
+ @brt.source.field.value = @brt.source.field.value.gsub(/\./, ',')
258
+ end
259
+
260
+ Then /^the target field is copied from the source field, but commas have been replaced by periods$/ do
261
+ source_field_value = @brt.source.field.value
262
+
263
+ @brt.run_transforms
264
+ expect(@brt.target.field.value).to eq source_field_value.gsub(/,/, '.')
265
+ end
266
+
267
+
268
+ ### Field presence
269
+
270
+ Then /^only the following fields should be present on the target:$/ do |table|
271
+ table.rows.each do |row|
272
+ field = row.first
273
+ step "the target field '#{field}'"
274
+ end
275
+
276
+ @brt.run_transforms
277
+ expect(@brt.target.data_obj.fields.keys).to match_array @brt.target.fields.names
278
+ end
279
+
280
+ ### Record-level expectations
281
+
282
+ Then /^the record should be (?i)(Retained|Rejected)(?-i)(?: without error|)$/ do |action|
283
+ source_size = @brt.source.size
284
+ @brt.run_transforms
285
+ targets_size = @brt.targets.total_size
286
+
287
+ case
288
+ when action.downcase == 'retained'
289
+ expect(targets_size).to eq source_size
290
+ when action.downcase == 'rejected'
291
+ expect(targets_size).to eq 0
292
+ else
293
+ raise "Unknown action #{action}"
294
+ end
295
+ end
296
+
297
+ Then /^the record should (not be|be) present on the target$/ do |action|
298
+ map_action = { 'not be' => 'rejected', 'be' => 'retained' }
299
+ step "the record should be #{map_action[action]}"
300
+ end
301
+
302
+ Then /^a target record is created$/ do
303
+ @brt.run_transforms
304
+ expect(@brt.targets.total_size).to be > 0
305
+ end
306
+
307
+ Then /^a target record is not created$/ do
308
+ @brt.run_transforms
309
+ expect(@brt.targets.total_size).to be 0
310
+ end