remi 0.2.14 → 0.2.15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/features/examples.feature +14 -0
- data/features/parameters.feature +24 -0
- data/features/step_definitions/remi_step.rb +55 -20
- data/features/support/env_app.rb +1 -0
- data/jobs/parameters_job.rb +23 -0
- data/lib/remi/cucumber/business_rules.rb +22 -11
- data/lib/remi/cucumber/data_source.rb +1 -1
- data/lib/remi/job.rb +1 -1
- data/lib/remi/version.rb +1 -1
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 89aa19a0e9852eb3fcbf63cd3890102501fc27ff
|
4
|
+
data.tar.gz: ef447ea96c09edd89953aabef5b9e18188d06c50
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 11db98f16aff0c37c83a02dd44dec3ec6e1f908c61e85aaeb277b789acc58b2eaf0706b10274591004da92e0ea66321b1ee2fe5318422772c251519150d41d80
|
7
|
+
data.tar.gz: 7639ef4a4b9b3aa03385e270b5aa3ea10796dd9e9250819e706be0fa366d95dcb4c886e6ccab54e5546f0aa7a38ea6adbb61909fa6919be3350aa68dc8d6b0c0
|
data/Gemfile.lock
CHANGED
data/features/examples.feature
CHANGED
@@ -47,3 +47,17 @@ Feature: This tests the creation of example records.
|
|
47
47
|
And the target has 3 records where 'Quantity' is greater than 0.9
|
48
48
|
And the target has 3 records where 'Quantity' is between 0 and 1
|
49
49
|
And the target has 2 records where 'Quantity' is between 0.6 and 2
|
50
|
+
|
51
|
+
|
52
|
+
|
53
|
+
Scenario: Adding data to a source that already has example data.
|
54
|
+
Given the following example for 'Source Data':
|
55
|
+
| Id | Name |
|
56
|
+
| 1 | Alpha |
|
57
|
+
| 2 | Beta |
|
58
|
+
| 3 | Gamma |
|
59
|
+
And the following records are appended to 'Source Data':
|
60
|
+
| Id | Name |
|
61
|
+
| 4 | Delta |
|
62
|
+
| 5 | Epsilon |
|
63
|
+
Then the target has 5 records
|
@@ -0,0 +1,24 @@
|
|
1
|
+
Feature: Tests setting parameters
|
2
|
+
|
3
|
+
Background:
|
4
|
+
Given the job is 'Parameters'
|
5
|
+
And the job source 'Source Data'
|
6
|
+
And the job target 'Target Data'
|
7
|
+
|
8
|
+
And the source 'Source Data'
|
9
|
+
And the target 'Target Data'
|
10
|
+
|
11
|
+
And the source field 'parameter_name' is set to the value "test_parameter"
|
12
|
+
|
13
|
+
Scenario: Setting a parameter explicitly.
|
14
|
+
Given the job parameter 'myparam' is "happypants"
|
15
|
+
Then the target field 'myparam' is set to the value "happypants"
|
16
|
+
|
17
|
+
Scenario: Dynamically retreiving a parameter that exists.
|
18
|
+
Given the source field 'parameter_name' is set to the value "test_parameter"
|
19
|
+
Then the target field 'parameter_name' is set to the value "my test parameter value"
|
20
|
+
|
21
|
+
@fails
|
22
|
+
Scenario: Dynamically retreiving a parameter that does not exists.
|
23
|
+
Given the source field 'parameter_name' is set to the value "nonexisting_parameter"
|
24
|
+
Then the target field 'parameter_name' is set to the value ""
|
@@ -15,16 +15,16 @@ Given /^the job target '([[:alnum:]\s\-_]+)'$/ do |arg|
|
|
15
15
|
@brt.add_job_target arg
|
16
16
|
end
|
17
17
|
|
18
|
-
Given /^the following example(?: record| records|) called '([[:alnum:]\s\-_]+)':$/ do |arg, example_table|
|
19
|
-
@brt.add_example arg, example_table
|
20
|
-
end
|
21
|
-
|
22
18
|
Given /^the job parameter '([[:alnum:]\s\-_]+)' is "([^"]*)"$/ do |param, value|
|
23
19
|
@brt.set_job_parameter(param, value)
|
24
20
|
end
|
25
21
|
|
26
22
|
### Setting up example data
|
27
23
|
|
24
|
+
Given /^the following example(?: record| records|) called '([[:alnum:]\s\-_]+)':$/ do |arg, example_table|
|
25
|
+
@brt.add_example arg, example_table
|
26
|
+
end
|
27
|
+
|
28
28
|
Given /^the following example(?: record| records|) for '([[:alnum:]\s\-_]+)':$/ do |source_name, example_table|
|
29
29
|
example_name = source_name
|
30
30
|
@brt.add_example example_name, example_table
|
@@ -35,6 +35,11 @@ Given /^the example '([[:alnum:]\s\-_]+)' for '([[:alnum:]\s\-_]+)'$/ do |exampl
|
|
35
35
|
@brt.job_sources[source_name].stub_data_with(@brt.examples[example_name])
|
36
36
|
end
|
37
37
|
|
38
|
+
Given /^the following (?:record is|records are) appended to '([[:alnum:]\s\-_]+)':$/ do |source_name, example_table|
|
39
|
+
example_name = SecureRandom.uuid
|
40
|
+
@brt.add_example example_name, example_table
|
41
|
+
@brt.job_sources[source_name].append_data_with(@brt.examples[example_name])
|
42
|
+
end
|
38
43
|
|
39
44
|
### Source file processing
|
40
45
|
|
@@ -106,8 +111,8 @@ Given /^the source '([[:alnum:]\s\-_]+)'$/ do |arg|
|
|
106
111
|
@brt.add_source(arg)
|
107
112
|
end
|
108
113
|
|
109
|
-
Given /^the source field '(
|
110
|
-
@brt.sources.add_field(
|
114
|
+
Given /^the source field '([^']+)'$/ do |source_field_name|
|
115
|
+
@brt.sources.add_field(source_field_name)
|
111
116
|
end
|
112
117
|
|
113
118
|
Given /^the source field has the value "([^"]*)"$/ do |arg|
|
@@ -120,7 +125,8 @@ end
|
|
120
125
|
|
121
126
|
When /^the source field '(.+)' (?:has an empty value|is blank)$/ do |source_field|
|
122
127
|
step "the source field '#{source_field}'"
|
123
|
-
@brt.
|
128
|
+
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
129
|
+
@brt.sources[source_name].fields[source_field_name].value = ''
|
124
130
|
end
|
125
131
|
|
126
132
|
Given /^the source field '([^:]+)' (?:has|is set to) the value "([^"]*)"$/ do |source_field, value|
|
@@ -130,20 +136,33 @@ end
|
|
130
136
|
|
131
137
|
Given /^the source field '(.+:.+)' (?:has|is set to) the value "([^"]*)"$/ do |source_field, value|
|
132
138
|
step "the source field '#{source_field}'"
|
133
|
-
source_name, field_name =
|
139
|
+
source_name, field_name = @brt.sources.parse_full_field(source_field)
|
134
140
|
@brt.sources[source_name].fields[field_name].value = Remi::BusinessRules::ParseFormula.parse(value)
|
135
141
|
end
|
136
142
|
|
143
|
+
Given /^the source field '(.+:.+)' (?:has|is set to) the value in the source field '(.+:.+)'$/ do |source_field, other_source_field|
|
144
|
+
step "the source field '#{source_field}'"
|
145
|
+
step "the source field '#{other_source_field}'"
|
146
|
+
source_name, field_name = @brt.sources.parse_full_field(source_field)
|
147
|
+
other_source_name, other_field_name = @brt.sources.parse_full_field(other_source_field)
|
148
|
+
|
149
|
+
@brt.sources[source_name].fields[field_name].value = @brt.sources[other_source_name].fields[other_field_name].value
|
150
|
+
end
|
151
|
+
|
137
152
|
Given /^the source field '(.+:.+)' (?:has|is set to) the value in the source field '(.+:.+)', prefixed with "([^"]*)"$/ do |source_field, other_source_field, prefix|
|
138
153
|
step "the source field '#{source_field}'"
|
139
154
|
step "the source field '#{other_source_field}'"
|
140
|
-
source_name, field_name =
|
141
|
-
other_source_name, other_field_name =
|
155
|
+
source_name, field_name = @brt.sources.parse_full_field(source_field)
|
156
|
+
other_source_name, other_field_name = @brt.sources.parse_full_field(other_source_field)
|
142
157
|
|
143
158
|
prefixed = "#{prefix}#{@brt.sources[other_source_name].fields[other_field_name].value}"
|
144
159
|
@brt.sources[source_name].fields[field_name].value = prefixed
|
145
160
|
end
|
146
161
|
|
162
|
+
Given /^the source data are tied through the fields '(.+:.+)' and '(.+:.+)'$/ do |source_field, other_source_field|
|
163
|
+
step "the source field '#{other_source_field}' is set to the value in the source field '#{source_field}'"
|
164
|
+
end
|
165
|
+
|
147
166
|
Given /^the source field is parsed with the date format "([^"]*)"$/ do |date_format|
|
148
167
|
expect(@brt.source.field.metadata[:format]).to eq date_format
|
149
168
|
end
|
@@ -180,10 +199,10 @@ Then /^the target field '(.+)' is copied from the source field '(.+:.+)'$/ do |t
|
|
180
199
|
step "the target field '#{target_field}'"
|
181
200
|
step "the source field '#{source_field}'"
|
182
201
|
|
183
|
-
source_name, source_field_name =
|
202
|
+
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
184
203
|
|
185
204
|
@brt.run_transforms
|
186
|
-
expect(@brt.target.
|
205
|
+
expect(@brt.target.fields[target_field].value).to eq (@brt.sources[source_name].fields[source_field_name].value)
|
187
206
|
end
|
188
207
|
|
189
208
|
Then /^the target field '(.+)' is copied from the source field '([^:]+)'$/ do |target_field, source_field|
|
@@ -200,9 +219,13 @@ Then /^the target field is (?:set to the value|populated with) "([^"]*)"$/ do |v
|
|
200
219
|
end
|
201
220
|
|
202
221
|
Then /^the target field '(.+)' is (?:set to the value|populated with) "([^"]*)"$/ do |target_field, value|
|
222
|
+
expect_cucumber {
|
223
|
+
expect {
|
203
224
|
@brt.targets.add_field(target_field)
|
204
225
|
@brt.run_transforms
|
226
|
+
}.not_to raise_error
|
205
227
|
expect(@brt.targets.fields[target_field].values.uniq).to eq [Remi::BusinessRules::ParseFormula.parse(value)]
|
228
|
+
}
|
206
229
|
end
|
207
230
|
|
208
231
|
Then /^the target field '(.+)' is in the list "([^"]*)"$/ do |target_field, list|
|
@@ -234,6 +257,15 @@ Then /^the target should match the example '([[:alnum:]\s]+)'$/ do |example_name
|
|
234
257
|
step "the target '#{target_name}' should match the example '#{example_name}'"
|
235
258
|
end
|
236
259
|
|
260
|
+
Then /^the target field '(.+)' is populated from the source field '(.+)' after translating it according to:$/ do |target_field, source_field, translation_table|
|
261
|
+
step "the target field '#{target_field}'"
|
262
|
+
|
263
|
+
translation_table.rows.each do |translation_row|
|
264
|
+
step "the source field '#{source_field}' is set to the value \"#{translation_row[0]}\""
|
265
|
+
@brt.run_transforms
|
266
|
+
expect(@brt.target.fields[target_field].value).to eq translation_row[1]
|
267
|
+
end
|
268
|
+
end
|
237
269
|
|
238
270
|
### Transforms
|
239
271
|
|
@@ -292,23 +324,26 @@ Then /^the target field '(.+)' is populated with "([^"]*)" using the format "([^
|
|
292
324
|
end
|
293
325
|
|
294
326
|
Then /^the target field '(.+)' is the first non-blank value from source fields '(.+)'$/ do |target_field_name, source_field_list|
|
295
|
-
|
296
|
-
field_with_quotes.match(/'(.+)'/)[1]
|
327
|
+
source_fields = "'#{source_field_list}'".split(',').map do |field_with_quotes|
|
328
|
+
full_field_name = field_with_quotes.match(/'(.+)'/)[1]
|
329
|
+
|
330
|
+
source_name, field_name = @brt.sources.parse_full_field(full_field_name)
|
331
|
+
{ full_field_name: full_field_name, source: source_name, field: field_name }
|
297
332
|
end
|
298
333
|
|
299
|
-
|
300
|
-
step "the source field '#{
|
334
|
+
source_fields.each do |source_field|
|
335
|
+
step "the source field '#{source_field[:full_field_name]}'"
|
301
336
|
end
|
302
337
|
step "the target field '#{target_field_name}'"
|
303
338
|
|
304
|
-
|
339
|
+
source_fields.each do |source_field|
|
305
340
|
@brt.run_transforms
|
306
341
|
|
307
|
-
source_values =
|
342
|
+
source_values = source_fields.map { |v_source_field| @brt.sources[v_source_field[:source]].fields[v_source_field[:field]].value }
|
308
343
|
source_values_nvl = source_values.find { |arg| !arg.blank? }
|
309
344
|
|
310
345
|
expect_cucumber { expect(@brt.target.fields[target_field_name].value).to eq source_values_nvl }
|
311
|
-
@brt.source.fields[
|
346
|
+
@brt.sources[source_field[:source]].fields[source_field[:field]].value = ''
|
312
347
|
end
|
313
348
|
|
314
349
|
end
|
@@ -384,7 +419,7 @@ end
|
|
384
419
|
|
385
420
|
Then /^the target '([[:alnum:]\s\-_]+)' has (\d+) record(?:s|)$/ do |target_name, nrecords|
|
386
421
|
@brt.run_transforms
|
387
|
-
expect(@brt.targets[target_name].size).to
|
422
|
+
expect(@brt.targets[target_name].size).to eq nrecords.to_i
|
388
423
|
end
|
389
424
|
|
390
425
|
Then /^the target has (\d+) record(?:s|) where '([[:alnum:]\s\-_]+)' is "([^"]*)"$/ do |nrecords, field_name, value|
|
data/features/support/env_app.rb
CHANGED
@@ -0,0 +1,23 @@
|
|
1
|
+
require_relative 'all_jobs_shared'
|
2
|
+
|
3
|
+
class ParametersJob
|
4
|
+
include AllJobsShared
|
5
|
+
|
6
|
+
define_param :test_parameter, "my test parameter value"
|
7
|
+
|
8
|
+
define_target :source_data, Remi::DataSource::DataFrame,
|
9
|
+
fields: {
|
10
|
+
:parameter_name => {}
|
11
|
+
}
|
12
|
+
define_target :target_data, Remi::DataTarget::DataFrame
|
13
|
+
|
14
|
+
define_transform :main do
|
15
|
+
Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
|
16
|
+
map source(nil) .target(:myparam)
|
17
|
+
.transform(Remi::Transform[:constant].(params[:myparam]))
|
18
|
+
map source(:parameter_name) .target(:parameter_name)
|
19
|
+
.transform(->(v) { params[v.to_sym] })
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
@@ -1,10 +1,6 @@
|
|
1
1
|
module Remi::BusinessRules
|
2
2
|
using Remi::Refinements::Symbolizer
|
3
3
|
|
4
|
-
def self.parse_full_field(full_field_name)
|
5
|
-
full_field_name.split(':').map(&:strip)
|
6
|
-
end
|
7
|
-
|
8
4
|
def self.csv_opt_map
|
9
5
|
{
|
10
6
|
'tab' => "\t",
|
@@ -173,18 +169,25 @@ module Remi::BusinessRules
|
|
173
169
|
@subjects.keys
|
174
170
|
end
|
175
171
|
|
172
|
+
def parse_full_field(full_field_name, multi: false)
|
173
|
+
if full_field_name.include? ':'
|
174
|
+
full_field_name.split(':').map(&:strip)
|
175
|
+
elsif multi
|
176
|
+
[@subjects.keys, full_field_name]
|
177
|
+
else
|
178
|
+
raise "Multiple subjects defined: #{keys}" unless @subjects.size == 1
|
179
|
+
[@subjects.keys.first, full_field_name]
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
176
183
|
def add_subject(subject_name, subject)
|
177
184
|
@subjects[subject_name] ||= DataSubject.new(subject)
|
178
185
|
end
|
179
186
|
|
180
187
|
def add_field(full_field_name)
|
181
|
-
|
182
|
-
|
188
|
+
subject_names, field_name = parse_full_field(full_field_name, multi: true)
|
189
|
+
Array(subject_names).each do |subject_name|
|
183
190
|
@subjects[subject_name].add_field(field_name)
|
184
|
-
else
|
185
|
-
@subjects.each do |subject_name, subject|
|
186
|
-
subject.add_field(full_field_name)
|
187
|
-
end
|
188
191
|
end
|
189
192
|
end
|
190
193
|
|
@@ -285,9 +288,17 @@ module Remi::BusinessRules
|
|
285
288
|
@data_obj.stub_df if @data_obj.respond_to? :stub_df
|
286
289
|
end
|
287
290
|
|
291
|
+
def example_to_df(example)
|
292
|
+
example.to_df(@data_obj.df.row[0].to_hash, field_symbolizer: @data_obj.field_symbolizer)
|
293
|
+
end
|
294
|
+
|
288
295
|
def stub_data_with(example)
|
289
296
|
stub_data
|
290
|
-
@data_obj.df = example
|
297
|
+
@data_obj.df = example_to_df(example)
|
298
|
+
end
|
299
|
+
|
300
|
+
def append_data_with(example)
|
301
|
+
@data_obj.df = @data_obj.df.concat example_to_df(example)
|
291
302
|
end
|
292
303
|
|
293
304
|
|
@@ -23,7 +23,7 @@ module Remi
|
|
23
23
|
float: ->() { Faker::Number.decimal(2,2) },
|
24
24
|
integer: ->() { Faker::Number.number(4) },
|
25
25
|
date: ->() { Faker::Date.backward(3650) },
|
26
|
-
datetime: ->() { Faker::Time.backward(3650) },
|
26
|
+
datetime: ->() { Faker::Time.backward(3650).to_datetime },
|
27
27
|
boolean: ->() { ['T','F'].shuffle.first }
|
28
28
|
})
|
29
29
|
end
|
data/lib/remi/job.rb
CHANGED
data/lib/remi/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: remi
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.15
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sterling Paramore
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-02-
|
11
|
+
date: 2016-02-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: daru
|
@@ -227,6 +227,7 @@ files:
|
|
227
227
|
- features/aggregate.feature
|
228
228
|
- features/examples.feature
|
229
229
|
- features/formulas.feature
|
230
|
+
- features/parameters.feature
|
230
231
|
- features/sample_job.feature
|
231
232
|
- features/step_definitions/remi_step.rb
|
232
233
|
- features/support/env.rb
|
@@ -238,6 +239,7 @@ files:
|
|
238
239
|
- jobs/aggregate_job.rb
|
239
240
|
- jobs/all_jobs_shared.rb
|
240
241
|
- jobs/copy_source_job.rb
|
242
|
+
- jobs/parameters_job.rb
|
241
243
|
- jobs/sample_job.rb
|
242
244
|
- jobs/transforms/date_diff_job.rb
|
243
245
|
- jobs/transforms/nvl_job.rb
|
@@ -300,6 +302,7 @@ test_files:
|
|
300
302
|
- features/aggregate.feature
|
301
303
|
- features/examples.feature
|
302
304
|
- features/formulas.feature
|
305
|
+
- features/parameters.feature
|
303
306
|
- features/sample_job.feature
|
304
307
|
- features/step_definitions/remi_step.rb
|
305
308
|
- features/support/env.rb
|