remi 0.0.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.bundle/config +2 -0
- data/.gitignore +3 -2
- data/.rspec +2 -0
- data/.ruby-version +1 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +123 -0
- data/LICENSE.txt +21 -0
- data/README.md +94 -3
- data/bin/remi +8 -0
- data/doc/install-rbenv-os_x.md +47 -0
- data/lib/remi.rb +56 -9
- data/lib/remi/cli.rb +56 -0
- data/lib/remi/core/daru.rb +28 -0
- data/lib/remi/core/refinements.rb +21 -0
- data/lib/remi/core/string.rb +8 -0
- data/lib/remi/cucumber.rb +7 -0
- data/lib/remi/cucumber/business_rules.rb +504 -0
- data/lib/remi/cucumber/data_source.rb +63 -0
- data/lib/remi/data_source.rb +13 -0
- data/lib/remi/data_source/csv_file.rb +79 -0
- data/lib/remi/data_source/data_frame.rb +10 -0
- data/lib/remi/data_source/postgres.rb +58 -0
- data/lib/remi/data_source/salesforce.rb +78 -0
- data/lib/remi/data_subject.rb +25 -0
- data/lib/remi/data_target.rb +15 -0
- data/lib/remi/data_target/csv_file.rb +49 -0
- data/lib/remi/data_target/data_frame.rb +14 -0
- data/lib/remi/data_target/salesforce.rb +49 -0
- data/lib/remi/extractor/sftp_file.rb +84 -0
- data/lib/remi/field_symbolizers.rb +17 -0
- data/lib/remi/job.rb +200 -0
- data/lib/remi/lookup/regex_sieve.rb +55 -0
- data/lib/remi/project/features/examples.feature +24 -0
- data/lib/remi/project/features/formulas.feature +64 -0
- data/lib/remi/project/features/sample_job.feature +304 -0
- data/lib/remi/project/features/step_definitions/remi_step.rb +310 -0
- data/lib/remi/project/features/support/env.rb +10 -0
- data/lib/remi/project/features/support/env_app.rb +3 -0
- data/lib/remi/project/features/transforms/date_diff.feature +50 -0
- data/lib/remi/project/features/transforms/parse_date.feature +34 -0
- data/lib/remi/project/features/transforms/prefix.feature +15 -0
- data/lib/remi/project/jobs/all_jobs_shared.rb +25 -0
- data/lib/remi/project/jobs/copy_source_job.rb +12 -0
- data/lib/remi/project/jobs/sample_job.rb +164 -0
- data/lib/remi/project/jobs/transforms/date_diff_job.rb +17 -0
- data/lib/remi/project/jobs/transforms/parse_date_job.rb +18 -0
- data/lib/remi/project/jobs/transforms/prefix_job.rb +16 -0
- data/lib/remi/project/jobs/transforms/transform_jobs.rb +3 -0
- data/lib/remi/settings.rb +39 -0
- data/lib/remi/sf_bulk_helper.rb +265 -0
- data/lib/remi/source_to_target_map.rb +93 -0
- data/lib/remi/transform.rb +137 -0
- data/lib/remi/version.rb +3 -0
- data/remi.gemspec +25 -7
- data/workbooks/sample_workbook.ipynb +56 -0
- data/workbooks/workbook_helper.rb +1 -0
- metadata +234 -17
- data/lib/noodling.rb +0 -163
- data/test/test_NAME.rb +0 -19
@@ -0,0 +1,304 @@
|
|
1
|
+
Feature: This is a sample feature file.
|
2
|
+
It demonstrates some of the functionality of using Remi for BRDD.
|
3
|
+
|
4
|
+
Background:
|
5
|
+
Given the job is 'Sample'
|
6
|
+
And the job source 'Sample File'
|
7
|
+
And the job source 'Existing Contacts'
|
8
|
+
And the job target 'Contact Updates'
|
9
|
+
And the job target 'Contact Creates'
|
10
|
+
|
11
|
+
And the following example record called 'existing contact':
|
12
|
+
| Id | External_ID__c |
|
13
|
+
| 003G0030H1BxH0xIIF | SAMP12345 |
|
14
|
+
|
15
|
+
And the following example record called 'sample records to load':
|
16
|
+
| Program | Student Id |
|
17
|
+
| BIO | 12345 |
|
18
|
+
| BIO | 0 |
|
19
|
+
|
20
|
+
And the following example record called 'sample record to update':
|
21
|
+
| Program | Student Id |
|
22
|
+
| BIO | 12345 |
|
23
|
+
|
24
|
+
And the following example record called 'sample record to create':
|
25
|
+
| Program | Student Id |
|
26
|
+
| BIO | 0 |
|
27
|
+
|
28
|
+
##### File Specifications #####
|
29
|
+
|
30
|
+
Scenario: Selecting and downloading the appropriate source files
|
31
|
+
|
32
|
+
Given the source 'Sample File'
|
33
|
+
And files with names matching the pattern /^SampleFile_(\d+)\.txt/
|
34
|
+
Then the file with the latest date stamp will be downloaded for processing
|
35
|
+
|
36
|
+
Given files with names that do not match the pattern /^SampleFile_(\d+)\.txt/
|
37
|
+
Then no files will be downloaded for processing
|
38
|
+
|
39
|
+
|
40
|
+
Scenario: In order to be parsed and properly processed, the file must conform
|
41
|
+
to expectations about its structure and content.
|
42
|
+
|
43
|
+
Given the source 'Sample File'
|
44
|
+
And the source file is delimited with a comma
|
45
|
+
And the source file is encoded using "ISO-8859-1" format
|
46
|
+
And the source file uses a double quote to quote embedded delimiters
|
47
|
+
And the source file uses a preceeding double quote to escape an embedded quoting character
|
48
|
+
And the source file uses windows or unix line endings
|
49
|
+
And the source file contains a header row
|
50
|
+
And the source file contains at least the following headers in no particular order:
|
51
|
+
| header |
|
52
|
+
| Student Id |
|
53
|
+
| School Id |
|
54
|
+
| School Name |
|
55
|
+
| Program |
|
56
|
+
| Last Name |
|
57
|
+
| First Name |
|
58
|
+
| Current Email |
|
59
|
+
| Mailing Address Line 1 |
|
60
|
+
| Mailing Address Line 2 |
|
61
|
+
| Mailing City |
|
62
|
+
| Mailing State |
|
63
|
+
| Mailing Postal Code |
|
64
|
+
| Birthdate |
|
65
|
+
| Applied Date |
|
66
|
+
|
67
|
+
|
68
|
+
##### Record Acceptance #####
|
69
|
+
|
70
|
+
Scenario Outline: 'Sample File' records are rejected for non-matching programs.
|
71
|
+
Records are rejected from all targets without error unless the
|
72
|
+
'Program' name is included in the list of acceptable program
|
73
|
+
names. The examples below are not exhaustive.
|
74
|
+
|
75
|
+
|
76
|
+
Given the source 'Sample File'
|
77
|
+
And the target 'Contact Updates'
|
78
|
+
And the target 'Contact Creates'
|
79
|
+
And the example 'sample records to load' for 'Sample File'
|
80
|
+
And the example 'existing contact' for 'Existing Contacts'
|
81
|
+
|
82
|
+
When the source field 'Program' has the value "<Program>"
|
83
|
+
Then the record should be <Action> without error
|
84
|
+
|
85
|
+
Examples:
|
86
|
+
| Program | Action |
|
87
|
+
| BIO | Retained |
|
88
|
+
| Fake Biology | Rejected |
|
89
|
+
| COMM | Rejected |
|
90
|
+
| CHEM | Retained |
|
91
|
+
|
92
|
+
|
93
|
+
##### Transformations common to both contact updates and creates #####
|
94
|
+
|
95
|
+
Scenario Outline: Populating Major__c for updates and creates.
|
96
|
+
This scenario uses the program name mapping. The examples below
|
97
|
+
are not exhaustive, but are intended for demonstration and testing
|
98
|
+
of edge cases.
|
99
|
+
|
100
|
+
Given the source 'Sample File'
|
101
|
+
And the target 'Contact Updates'
|
102
|
+
And the target 'Contact Creates'
|
103
|
+
And the example 'existing contact' for 'Existing Contacts'
|
104
|
+
And the example 'sample records to load' for 'Sample File'
|
105
|
+
|
106
|
+
When the source field 'Program' has the value "<Program>"
|
107
|
+
Then the target field 'Major__c' is set to the value "<Major__c>"
|
108
|
+
|
109
|
+
Examples:
|
110
|
+
| Program | Major__c |
|
111
|
+
| BIO | Biology |
|
112
|
+
| CHEM | Chemistry |
|
113
|
+
| Chemistry | Chemistry |
|
114
|
+
| Physical Chemistry | Chemistry |
|
115
|
+
| Microbiology | Biology |
|
116
|
+
|
117
|
+
|
118
|
+
##### Transformations for creating contact records #####
|
119
|
+
|
120
|
+
Scenario: Fields that are present on create
|
121
|
+
Given the target 'Contact Creates'
|
122
|
+
And the example 'existing contact' for 'Existing Contacts'
|
123
|
+
And the example 'sample record to create' for 'Sample File'
|
124
|
+
|
125
|
+
Then only the following fields should be present on the target:
|
126
|
+
| Field Name |
|
127
|
+
| External_ID__c |
|
128
|
+
| School_ID__c |
|
129
|
+
| School_Name__c |
|
130
|
+
| School__c |
|
131
|
+
| Major__c |
|
132
|
+
| FirstName |
|
133
|
+
| LastName |
|
134
|
+
| Email |
|
135
|
+
| MailingStreet |
|
136
|
+
| MailingCity |
|
137
|
+
| MailingState |
|
138
|
+
| MailingPostalCode |
|
139
|
+
| Birthdate |
|
140
|
+
| Applied_Date__c |
|
141
|
+
|
142
|
+
|
143
|
+
Scenario: Records that are directed to Contact creates
|
144
|
+
This example shows how to create a scenario that simultaneously references
|
145
|
+
two sources.
|
146
|
+
|
147
|
+
Given the source 'Sample File'
|
148
|
+
And the source 'Existing Contacts'
|
149
|
+
And the target 'Contact Creates'
|
150
|
+
And the example 'sample record to create' for 'Sample File'
|
151
|
+
And the example 'existing contact' for 'Existing Contacts'
|
152
|
+
|
153
|
+
And the source field 'Sample File: Student Id'
|
154
|
+
And the source field 'Existing Contacts: External_ID__c' has the value "some arbitrary value"
|
155
|
+
Then a target record is created
|
156
|
+
|
157
|
+
When the source field 'Existing Contacts: External_ID__c' has the value in the source field 'Sample File: Student Id', prefixed with "SAMP"
|
158
|
+
Then a target record is not created
|
159
|
+
|
160
|
+
|
161
|
+
Scenario Outline: Fields that are copied from the 'Sample File' to 'Contact Creates'
|
162
|
+
This example shows how a scenario can be used to specify very simple
|
163
|
+
field mappings, like a simple copy with missing value handling.
|
164
|
+
|
165
|
+
Given the source 'Sample File'
|
166
|
+
And the target 'Contact Creates'
|
167
|
+
And the example 'sample record to create' for 'Sample File'
|
168
|
+
|
169
|
+
And the source field '<Source Field>'
|
170
|
+
Then the target field '<Target Field>' is copied from the source field
|
171
|
+
|
172
|
+
When the source field is blank
|
173
|
+
Then the target field '<Target Field>' is populated with "<If Blank>"
|
174
|
+
|
175
|
+
Examples:
|
176
|
+
| Source Field | Target Field | If Blank |
|
177
|
+
| School Id | School_ID__c | |
|
178
|
+
| School Name | School_Name__c | |
|
179
|
+
| First Name | FirstName | Not Provided |
|
180
|
+
| Last Name | LastName | Not Provided |
|
181
|
+
| Mailing City | MailingCity | |
|
182
|
+
| Mailing State | MailingState | |
|
183
|
+
| Mailing Postal Code | MailingPostalCode | |
|
184
|
+
|
185
|
+
|
186
|
+
Scenario Outline: Date fields that are parsed and then copied from the Sample File
|
187
|
+
to 'Contact Creates'
|
188
|
+
|
189
|
+
Given the source 'Sample File'
|
190
|
+
And the target 'Contact Creates'
|
191
|
+
And the example 'sample record to create' for 'Sample File'
|
192
|
+
|
193
|
+
And the source field '<Source Field>'
|
194
|
+
And the source field is parsed with the date format "<Source Format>"
|
195
|
+
Then the target field '<Target Field>' is populated from the source field using the format "<Target Format>"
|
196
|
+
|
197
|
+
When the source field is blank
|
198
|
+
Then the target field '<Target Field>' is populated with "<If Blank>" using the format "<Target Format>"
|
199
|
+
|
200
|
+
Examples:
|
201
|
+
| Source Field | Source Format | Target Field | Target Format | If Blank |
|
202
|
+
| Applied Date | %m/%d/%Y | Applied_Date__c | %Y-%m-%d | *Today's Date* |
|
203
|
+
| Birthdate | %m/%d/%Y | Birthdate | %Y-%m-%d | |
|
204
|
+
|
205
|
+
|
206
|
+
Scenario: Populating School__c
|
207
|
+
This is an example of creating a field that is a delimited
|
208
|
+
concatenation of two fields, where we replace blank values with "Unknowns"
|
209
|
+
|
210
|
+
Given the source 'Sample File'
|
211
|
+
And the target 'Contact Creates'
|
212
|
+
And the example 'existing contact' for 'Existing Contacts'
|
213
|
+
And the example 'sample records to load' for 'Sample File'
|
214
|
+
|
215
|
+
And the source field 'School Id'
|
216
|
+
And the source field 'School Name'
|
217
|
+
And the target field 'School__c'
|
218
|
+
Then the target field is a concatenation of the source fields, delimited by "-"
|
219
|
+
|
220
|
+
When the source field 'School Id' is blank
|
221
|
+
And the source field 'School Name' has the value "some arbitrary value"
|
222
|
+
Then the target field is a concatenation of "Unknown" and 'School Name', delimited by "-"
|
223
|
+
|
224
|
+
When the source field 'School Id' has the value "some arbitrary value"
|
225
|
+
And the source field 'School Name' is blank
|
226
|
+
Then the target field is a concatenation of 'School Id' and "Unknown", delimited by "-"
|
227
|
+
|
228
|
+
|
229
|
+
Scenario: Populating Email
|
230
|
+
This example shows some pre-cleaning of e-mail addresses (replacing commas with periods)
|
231
|
+
|
232
|
+
Given the source 'Sample File'
|
233
|
+
And the target 'Contact Creates'
|
234
|
+
And the example 'sample record to create' for 'Sample File'
|
235
|
+
|
236
|
+
And the source field 'Current Email'
|
237
|
+
And the target field 'Email'
|
238
|
+
|
239
|
+
When the source field is a valid email address
|
240
|
+
Then the target field is copied from the source field
|
241
|
+
|
242
|
+
When the source field is not a valid email address
|
243
|
+
Then the target field is populated with ""
|
244
|
+
|
245
|
+
When the source field is a valid email address
|
246
|
+
And in the source field, periods have been used in place of commas
|
247
|
+
Then the target field is copied from the source field, but commas have been replaced by periods
|
248
|
+
|
249
|
+
|
250
|
+
Scenario: Populating External_ID__c
|
251
|
+
|
252
|
+
Given the source 'Sample File'
|
253
|
+
And the target 'Contact Creates'
|
254
|
+
And the example 'sample record to create' for 'Sample File'
|
255
|
+
|
256
|
+
And the source field 'Student Id'
|
257
|
+
And the target field 'External_ID__c'
|
258
|
+
Then the source field is prefixed with "SAMP" and loaded into the target field
|
259
|
+
|
260
|
+
|
261
|
+
Scenario: Populating MailingStreet
|
262
|
+
Given the source 'Sample File'
|
263
|
+
And the target 'Contact Creates'
|
264
|
+
And the example 'sample record to create' for 'Sample File'
|
265
|
+
|
266
|
+
And the source field 'Mailing Address Line 1'
|
267
|
+
And the source field 'Mailing Address Line 2'
|
268
|
+
And the target field 'MailingStreet'
|
269
|
+
Then the target field is a concatenation of the source fields, delimited by ", "
|
270
|
+
|
271
|
+
When the source field 'Mailing Address Line 1' is blank
|
272
|
+
Then the target field is populated with ""
|
273
|
+
|
274
|
+
|
275
|
+
##### Transformations for updating contact records #####
|
276
|
+
|
277
|
+
Scenario: Fields that are present on update
|
278
|
+
The business rules specify that once a student record has been
|
279
|
+
created, only updates to the program are accepted (all other
|
280
|
+
changes to be done in Salesforce)
|
281
|
+
|
282
|
+
Given the target 'Contact Updates'
|
283
|
+
And the example 'existing contact' for 'Existing Contacts'
|
284
|
+
And the example 'sample record to create' for 'Sample File'
|
285
|
+
|
286
|
+
Then only the following fields should be present on the target:
|
287
|
+
| Field Name |
|
288
|
+
| Id |
|
289
|
+
| Major__c |
|
290
|
+
|
291
|
+
Scenario: Records that are directed to Contact updates
|
292
|
+
|
293
|
+
Given the source 'Sample File'
|
294
|
+
And the source 'Existing Contacts'
|
295
|
+
And the target 'Contact Updates'
|
296
|
+
And the example 'sample record to update' for 'Sample File'
|
297
|
+
And the example 'existing contact' for 'Existing Contacts'
|
298
|
+
|
299
|
+
And the source field 'Sample File: Student Id'
|
300
|
+
And the source field 'Existing Contacts: External_ID__c' has the value in the source field 'Sample File: Student Id', prefixed with "SAMP"
|
301
|
+
Then a target record is created
|
302
|
+
|
303
|
+
When the source field 'Existing Contacts: External_ID__c' has the value "some arbitrary value"
|
304
|
+
Then a target record is not created
|
@@ -0,0 +1,310 @@
|
|
1
|
+
# Auto-generated from Remi. Editing this file is not recommended. Instead,
|
2
|
+
# create another step definition file to contain application-specific steps.
|
3
|
+
|
4
|
+
### Job and background setup
|
5
|
+
|
6
|
+
Given /^the job is '([[:alnum:]\s]+)'$/ do |arg|
|
7
|
+
@brt = Remi::BusinessRules::Tester.new(arg)
|
8
|
+
end
|
9
|
+
|
10
|
+
Given /^the job source '([[:alnum:]\s\-_]+)'$/ do |arg|
|
11
|
+
@brt.add_job_source arg
|
12
|
+
end
|
13
|
+
|
14
|
+
Given /^the job target '([[:alnum:]\s\-_]+)'$/ do |arg|
|
15
|
+
@brt.add_job_target arg
|
16
|
+
end
|
17
|
+
|
18
|
+
Given /^the following example record called '([[:alnum:]\s\-_]+)':$/ do |arg, example_table|
|
19
|
+
@brt.add_example arg, example_table
|
20
|
+
end
|
21
|
+
|
22
|
+
Given /^the job parameter '([[:alnum:]\s\-_]+)' is "([^"]*)"$/ do |param, value|
|
23
|
+
@brt.set_job_parameter(param, value)
|
24
|
+
end
|
25
|
+
|
26
|
+
### Setting up example data
|
27
|
+
|
28
|
+
Given /^the following example record for '([[:alnum:]\s\-_]+)':$/ do |source_name, example_table|
|
29
|
+
example_name = source_name
|
30
|
+
@brt.add_example example_name, example_table
|
31
|
+
@brt.job_sources[source_name].stub_data_with(@brt.examples[example_name])
|
32
|
+
end
|
33
|
+
|
34
|
+
Given /^the example '([[:alnum:]\s\-_]+)' for '([[:alnum:]\s\-_]+)'$/ do |example_name, source_name|
|
35
|
+
@brt.job_sources[source_name].stub_data_with(@brt.examples[example_name])
|
36
|
+
end
|
37
|
+
|
38
|
+
|
39
|
+
### Source file processing
|
40
|
+
|
41
|
+
Given /^files with names matching the pattern \/(.*)\/$/ do |pattern|
|
42
|
+
@brt.filestore.pattern(Regexp.new(pattern))
|
43
|
+
end
|
44
|
+
|
45
|
+
Given /^files with names that do not match the pattern \/(.*)\/$/ do |pattern|
|
46
|
+
@brt.filestore.anti_pattern(Regexp.new(pattern))
|
47
|
+
end
|
48
|
+
|
49
|
+
Given /^files delivered within the last (\d+) hours$/ do |hours|
|
50
|
+
@brt.filestore.delivered_since(Time.now - hours.to_i * 3600)
|
51
|
+
end
|
52
|
+
|
53
|
+
Given /^files were delivered more than (\d+) hours ago$/ do |hours|
|
54
|
+
@brt.filestore.delivered_before(Time.now - hours.to_i * 3600)
|
55
|
+
end
|
56
|
+
|
57
|
+
Then /^the file with the latest date stamp will be downloaded for processing$/ do
|
58
|
+
@brt.filestore.generate
|
59
|
+
@brt.source.mock_extractor(@brt.filestore)
|
60
|
+
expect(@brt.source.extract).to match_array Array(@brt.filestore.latest)
|
61
|
+
end
|
62
|
+
|
63
|
+
Then /^no files will be downloaded for processing$/ do
|
64
|
+
@brt.filestore.generate
|
65
|
+
@brt.source.mock_extractor(@brt.filestore)
|
66
|
+
expect { @brt.source.extract }.to raise_error Remi::Extractor::SftpFile::FileNotFoundError
|
67
|
+
end
|
68
|
+
|
69
|
+
|
70
|
+
Given /^the source file is delimited with a (\w+)$/ do |delimiter|
|
71
|
+
expect(@brt.source.csv_options[:col_sep]).to eq Remi::BusinessRules.csv_opt_map[delimiter]
|
72
|
+
end
|
73
|
+
|
74
|
+
Given /^the source file is encoded using "([^"]+)" format$/ do |encoding|
|
75
|
+
expect(@brt.source.csv_options[:encoding].split(':').first).to eq encoding
|
76
|
+
end
|
77
|
+
|
78
|
+
Given /^the source file uses a ([\w ]+) to quote embedded delimiters$/ do |quote_char|
|
79
|
+
expect(@brt.source.csv_options[:quote_char]).to eq Remi::BusinessRules.csv_opt_map[quote_char]
|
80
|
+
end
|
81
|
+
|
82
|
+
Given /^the source file uses a preceeding ([\w ]+) to escape an embedded quoting character$/ do |escape_char|
|
83
|
+
expect(@brt.source.csv_options[:quote_char]).to eq Remi::BusinessRules.csv_opt_map[escape_char]
|
84
|
+
end
|
85
|
+
|
86
|
+
Given /^the source file uses ([\w ]+) line endings$/ do |line_endings|
|
87
|
+
expect(@brt.source.csv_options[:row_sep]).to eq Remi::BusinessRules.csv_opt_map[line_endings]
|
88
|
+
end
|
89
|
+
|
90
|
+
Given /^the source file (contains|does not contain) a header row$/ do |header|
|
91
|
+
expect(@brt.source.csv_options[:headers]).to eq (header == 'contains')
|
92
|
+
end
|
93
|
+
|
94
|
+
Given /^the source file contains at least the following headers in no particular order:$/ do |table|
|
95
|
+
table.rows.each do |row|
|
96
|
+
field = row.first
|
97
|
+
step "the source field '#{field}'"
|
98
|
+
end
|
99
|
+
expect(@brt.source.data_obj.fields.keys).to include(*@brt.source.fields.names)
|
100
|
+
end
|
101
|
+
|
102
|
+
|
103
|
+
### Source
|
104
|
+
|
105
|
+
Given /^the source '([[:alnum:]\s\-_]+)'$/ do |arg|
|
106
|
+
@brt.add_source(arg)
|
107
|
+
end
|
108
|
+
|
109
|
+
Given /^the source field '(.+)'$/ do |arg|
|
110
|
+
@brt.sources.add_field(arg)
|
111
|
+
end
|
112
|
+
|
113
|
+
Given /^the source field has the value "([^"]*)"$/ do |arg|
|
114
|
+
@brt.source.field.value = Remi::BusinessRules::ParseFormula.parse(arg)
|
115
|
+
end
|
116
|
+
|
117
|
+
When /^the source field (?:has an empty value|is blank)$/ do
|
118
|
+
@brt.source.field.value = ''
|
119
|
+
end
|
120
|
+
|
121
|
+
When /^the source field '(.+)' (?:has an empty value|is blank)$/ do |source_field|
|
122
|
+
step "the source field '#{source_field}'"
|
123
|
+
@brt.source.fields[source_field].value = ''
|
124
|
+
end
|
125
|
+
|
126
|
+
Given /^the source field '([^:]+)' (?:has|is set to) the value "([^"]*)"$/ do |source_field, value|
|
127
|
+
step "the source field '#{source_field}'"
|
128
|
+
@brt.source.fields[source_field].value = Remi::BusinessRules::ParseFormula.parse(value)
|
129
|
+
end
|
130
|
+
|
131
|
+
Given /^the source field '(.+:.+)' (?:has|is set to) the value "([^"]*)"$/ do |source_field, value|
|
132
|
+
step "the source field '#{source_field}'"
|
133
|
+
source_name, field_name = *Remi::BusinessRules.parse_full_field(source_field)
|
134
|
+
@brt.sources[source_name].fields[field_name].value = Remi::BusinessRules::ParseFormula.parse(value)
|
135
|
+
end
|
136
|
+
|
137
|
+
Given /^the source field '(.+:.+)' (?:has|is set to) the value in the source field '(.+:.+)', prefixed with "([^"]*)"$/ do |source_field, other_source_field, prefix|
|
138
|
+
step "the source field '#{source_field}'"
|
139
|
+
step "the source field '#{other_source_field}'"
|
140
|
+
source_name, field_name = *Remi::BusinessRules.parse_full_field(source_field)
|
141
|
+
other_source_name, other_field_name = *Remi::BusinessRules.parse_full_field(other_source_field)
|
142
|
+
|
143
|
+
prefixed = "#{prefix}#{@brt.sources[other_source_name].fields[other_field_name].value}"
|
144
|
+
@brt.sources[source_name].fields[field_name].value = prefixed
|
145
|
+
end
|
146
|
+
|
147
|
+
Given /^the source field is parsed with the date format "([^"]*)"$/ do |date_format|
|
148
|
+
expect(@brt.source.field.metadata[:format]).to eq date_format
|
149
|
+
end
|
150
|
+
|
151
|
+
Given /^the source field is a valid email address$/ do
|
152
|
+
@brt.source.field.value = 'valid@example.com'
|
153
|
+
end
|
154
|
+
|
155
|
+
Given /^the source field is not a valid email address$/ do
|
156
|
+
@brt.source.field.value = 'invalid!example.com'
|
157
|
+
end
|
158
|
+
|
159
|
+
### Target
|
160
|
+
|
161
|
+
Given /^the target '([[:alnum:]\s\-_]+)'$/ do |arg|
|
162
|
+
@brt.add_target(arg)
|
163
|
+
end
|
164
|
+
|
165
|
+
Given /^the target field '([^']+)'$/ do |arg|
|
166
|
+
@brt.targets.add_field(arg)
|
167
|
+
end
|
168
|
+
|
169
|
+
Then /^the target field '(.+)' is copied from the source field$/ do |arg|
|
170
|
+
step "the target field '#{arg}'"
|
171
|
+
step "the target field is copied from the source field"
|
172
|
+
end
|
173
|
+
|
174
|
+
Then /^the target field is copied from the source field$/ do
|
175
|
+
@brt.run_transforms
|
176
|
+
expect(@brt.target.field.value).to eq (@brt.source.field.value)
|
177
|
+
end
|
178
|
+
|
179
|
+
Then /^the target field '(.+)' is copied from the source field '(.+:.+)'$/ do |target_field, source_field|
|
180
|
+
step "the target field '#{target_field}'"
|
181
|
+
step "the source field '#{source_field}'"
|
182
|
+
|
183
|
+
source_name, source_field_name = *Remi::BusinessRules.parse_full_field(source_field)
|
184
|
+
|
185
|
+
@brt.run_transforms
|
186
|
+
expect(@brt.target.field.value).to eq (@brt.sources[source_name].fields[source_field_name].value)
|
187
|
+
end
|
188
|
+
|
189
|
+
Then /^the target field is (?:set to the value|populated with) "([^"]*)"$/ do |value|
|
190
|
+
@brt.run_transforms
|
191
|
+
expect(@brt.target.field.value).to eq Remi::BusinessRules::ParseFormula.parse(value)
|
192
|
+
end
|
193
|
+
|
194
|
+
Then /^the target field '(.+)' is (?:set to the value|populated with) "([^"]*)"$/ do |target_field, value|
|
195
|
+
@brt.targets.add_field(target_field)
|
196
|
+
@brt.run_transforms
|
197
|
+
expect(@brt.targets.fields[target_field].values.uniq).to eq [Remi::BusinessRules::ParseFormula.parse(value)]
|
198
|
+
end
|
199
|
+
|
200
|
+
Then /^the target field '(.+)' is the date (.+)$/ do |target_field, date_reference|
|
201
|
+
step "the target field '#{target_field}' is set to the value \"*#{date_reference}*\""
|
202
|
+
end
|
203
|
+
|
204
|
+
|
205
|
+
### Transforms
|
206
|
+
|
207
|
+
Then /^the target field is a concatenation of the source fields, delimited by "([^"]*)"$/ do |delimiter|
|
208
|
+
concatenated_source = @brt.sources.fields.values.uniq.map do |row|
|
209
|
+
Array(row.join(delimiter))
|
210
|
+
end
|
211
|
+
|
212
|
+
@brt.run_transforms
|
213
|
+
expect(@brt.targets.fields.values.uniq).to eq concatenated_source
|
214
|
+
end
|
215
|
+
|
216
|
+
Then /^the target field is a concatenation of "([^"]*)" and '(.+)', delimited by "([^"]*)"$/ do |constant, source_field, delimiter|
|
217
|
+
expected_value = [constant, @brt.sources.fields[source_field].value].join(delimiter)
|
218
|
+
@brt.run_transforms
|
219
|
+
expect(@brt.targets.fields.values.uniq).to eq [[expected_value]]
|
220
|
+
end
|
221
|
+
|
222
|
+
Then /^the target field is a concatenation of '(.+)' and "([^"]*)", delimited by "([^"]*)"$/ do |source_field, constant, delimiter|
|
223
|
+
expected_value = [@brt.sources.fields[source_field].value, constant].join(delimiter)
|
224
|
+
@brt.run_transforms
|
225
|
+
expect(@brt.targets.fields.values.uniq).to eq [[expected_value]]
|
226
|
+
end
|
227
|
+
|
228
|
+
Then /^the source field is prefixed with "([^"]*)" and loaded into the target field$/ do |prefix|
|
229
|
+
prefixed_source = "#{prefix}#{@brt.source.field.value}"
|
230
|
+
@brt.run_transforms
|
231
|
+
expect(@brt.target.field.value).to eq prefixed_source
|
232
|
+
end
|
233
|
+
|
234
|
+
Then /^the target field '(.+)' is populated from the source field using the format "([^"]*)"$/ do |target_field, target_format|
|
235
|
+
source_format = @brt.source.field.metadata[:format]
|
236
|
+
source_reformatted = Remi::Transform[:format_date].(from_fmt: source_format, to_fmt: target_format)
|
237
|
+
.call(@brt.source.field.value)
|
238
|
+
|
239
|
+
step "the target field '#{target_field}'"
|
240
|
+
@brt.run_transforms
|
241
|
+
expect(@brt.target.field.value).to eq source_reformatted
|
242
|
+
end
|
243
|
+
|
244
|
+
Then /^the target field '(.+)' is populated with "([^"]*)" using the format "([^"]*)"$/ do |target_field, target_value, target_format|
|
245
|
+
source_format = @brt.source.field.metadata[:format]
|
246
|
+
target_value_source_format = target_value == "*Today's Date*" ? Date.today.strftime(source_format) : target_value
|
247
|
+
target_reformatted = Remi::Transform[:format_date].(from_fmt: source_format, to_fmt: target_format)
|
248
|
+
.call(target_value_source_format)
|
249
|
+
|
250
|
+
step "the target field '#{target_field}'"
|
251
|
+
@brt.run_transforms
|
252
|
+
expect(@brt.target.field.value).to eq target_reformatted
|
253
|
+
end
|
254
|
+
|
255
|
+
|
256
|
+
When /^in the source field, periods have been used in place of commas$/ do
|
257
|
+
@brt.source.field.value = @brt.source.field.value.gsub(/\./, ',')
|
258
|
+
end
|
259
|
+
|
260
|
+
Then /^the target field is copied from the source field, but commas have been replaced by periods$/ do
|
261
|
+
source_field_value = @brt.source.field.value
|
262
|
+
|
263
|
+
@brt.run_transforms
|
264
|
+
expect(@brt.target.field.value).to eq source_field_value.gsub(/,/, '.')
|
265
|
+
end
|
266
|
+
|
267
|
+
|
268
|
+
### Field presence
|
269
|
+
|
270
|
+
Then /^only the following fields should be present on the target:$/ do |table|
|
271
|
+
table.rows.each do |row|
|
272
|
+
field = row.first
|
273
|
+
step "the target field '#{field}'"
|
274
|
+
end
|
275
|
+
|
276
|
+
@brt.run_transforms
|
277
|
+
expect(@brt.target.data_obj.fields.keys).to match_array @brt.target.fields.names
|
278
|
+
end
|
279
|
+
|
280
|
+
### Record-level expectations
|
281
|
+
|
282
|
+
Then /^the record should be (?i)(Retained|Rejected)(?-i)(?: without error|)$/ do |action|
|
283
|
+
source_size = @brt.source.size
|
284
|
+
@brt.run_transforms
|
285
|
+
targets_size = @brt.targets.total_size
|
286
|
+
|
287
|
+
case
|
288
|
+
when action.downcase == 'retained'
|
289
|
+
expect(targets_size).to eq source_size
|
290
|
+
when action.downcase == 'rejected'
|
291
|
+
expect(targets_size).to eq 0
|
292
|
+
else
|
293
|
+
raise "Unknown action #{action}"
|
294
|
+
end
|
295
|
+
end
|
296
|
+
|
297
|
+
Then /^the record should (not be|be) present on the target$/ do |action|
|
298
|
+
map_action = { 'not be' => 'rejected', 'be' => 'retained' }
|
299
|
+
step "the record should be #{map_action[action]}"
|
300
|
+
end
|
301
|
+
|
302
|
+
Then /^a target record is created$/ do
|
303
|
+
@brt.run_transforms
|
304
|
+
expect(@brt.targets.total_size).to be > 0
|
305
|
+
end
|
306
|
+
|
307
|
+
Then /^a target record is not created$/ do
|
308
|
+
@brt.run_transforms
|
309
|
+
expect(@brt.targets.total_size).to be 0
|
310
|
+
end
|