remi 0.2.27 → 0.2.28
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +3 -0
- data/Gemfile.lock +34 -5
- data/features/metadata.feature +17 -0
- data/features/step_definitions/remi_step.rb +6 -6
- data/features/transforms/date_diff.feature +1 -0
- data/jobs/aggregate_job.rb +0 -1
- data/jobs/all_jobs_shared.rb +0 -2
- data/jobs/copy_source_job.rb +0 -1
- data/jobs/csv_file_target_job.rb +0 -1
- data/jobs/metadata_job.rb +60 -0
- data/jobs/parameters_job.rb +1 -1
- data/jobs/sample_job.rb +19 -20
- data/jobs/sftp_file_target_job.rb +0 -1
- data/jobs/transforms/date_diff_job.rb +1 -1
- data/jobs/transforms/nvl_job.rb +1 -1
- data/jobs/transforms/parse_date_job.rb +7 -4
- data/jobs/transforms/prefix_job.rb +1 -1
- data/jobs/transforms/truncate_job.rb +1 -1
- data/lib/remi.rb +10 -15
- data/lib/remi/cucumber/business_rules.rb +23 -23
- data/lib/remi/cucumber/data_source.rb +2 -1
- data/lib/remi/data_frame.rb +36 -0
- data/lib/remi/data_frame/daru.rb +67 -0
- data/lib/remi/data_subject.rb +71 -10
- data/lib/remi/data_subject/csv_file.rb +151 -0
- data/lib/remi/data_subject/data_frame.rb +53 -0
- data/lib/remi/data_subject/postgres.rb +136 -0
- data/lib/remi/data_subject/salesforce.rb +136 -0
- data/lib/remi/data_subject/sftp_file.rb +66 -0
- data/lib/remi/fields.rb +8 -0
- data/lib/remi/source_to_target_map.rb +56 -32
- data/lib/remi/transform.rb +426 -83
- data/lib/remi/version.rb +1 -1
- data/remi.gemspec +2 -1
- data/spec/metadata_spec.rb +62 -0
- metadata +15 -28
- data/lib/remi/data_source.rb +0 -13
- data/lib/remi/data_source/csv_file.rb +0 -101
- data/lib/remi/data_source/data_frame.rb +0 -16
- data/lib/remi/data_source/postgres.rb +0 -58
- data/lib/remi/data_source/salesforce.rb +0 -87
- data/lib/remi/data_target.rb +0 -15
- data/lib/remi/data_target/csv_file.rb +0 -42
- data/lib/remi/data_target/data_frame.rb +0 -14
- data/lib/remi/data_target/postgres.rb +0 -74
- data/lib/remi/data_target/salesforce.rb +0 -54
- data/lib/remi/data_target/sftp_file.rb +0 -54
- data/lib/remi/refinements/daru.rb +0 -85
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: dc30c0a301e3be2a32b11febab47f55037a2d621
|
4
|
+
data.tar.gz: 2474212de553dafd2fb51ab30643ba65d30ec908
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 50dea06798dd136475c4a0401be65d8b19d1f0b8c82b2841e9551f95de6fc31897c55e4be5b58065e8cd792d2bfd9e37bd24a2001eddfbea8da44428bb32b1f2
|
7
|
+
data.tar.gz: ba9d970c2eecd2aba4bbd5467b25ae60bbc6827fcd36b1b89bbaacc301125d7e09af583874a0df0e9172dce1228934a7eff70aab641e53036909a06e38ee5e70
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,11 +1,28 @@
|
|
1
|
+
GIT
|
2
|
+
remote: git@github.com:inside-track/daru.git
|
3
|
+
revision: ad71b644c8e26789e3ab2258f046d5bc68b8df71
|
4
|
+
branch: itk-master
|
5
|
+
specs:
|
6
|
+
daru (0.1.2)
|
7
|
+
reportbuilder (~> 1.4)
|
8
|
+
spreadsheet (~> 1.1.1)
|
9
|
+
|
10
|
+
GIT
|
11
|
+
remote: git@github.com:inside-track/salesforce_bulk_api.git
|
12
|
+
revision: f597f06e5aa0cec3f9072b23fa09769676ec73da
|
13
|
+
branch: master
|
14
|
+
specs:
|
15
|
+
salesforce_bulk_api (0.0.12)
|
16
|
+
json
|
17
|
+
xml-simple
|
18
|
+
|
1
19
|
PATH
|
2
20
|
remote: .
|
3
21
|
specs:
|
4
|
-
remi (0.2.
|
22
|
+
remi (0.2.28)
|
5
23
|
activesupport (~> 4.2)
|
6
24
|
bond (~> 0.5)
|
7
25
|
cucumber (~> 2.1)
|
8
|
-
daru (= 0.1.2)
|
9
26
|
docile (~> 1.1)
|
10
27
|
faker (~> 1.6)
|
11
28
|
net-sftp (~> 2.1)
|
@@ -37,14 +54,16 @@ GEM
|
|
37
54
|
cucumber-core (1.4.0)
|
38
55
|
gherkin (~> 3.2.0)
|
39
56
|
cucumber-wire (0.0.1)
|
40
|
-
daru (0.1.2)
|
41
|
-
reportbuilder (~> 1.4)
|
42
|
-
spreadsheet (~> 1.1.1)
|
43
57
|
diff-lcs (1.2.5)
|
44
58
|
docile (1.1.5)
|
45
59
|
faker (1.6.3)
|
46
60
|
i18n (~> 0.5)
|
61
|
+
faraday (0.9.2)
|
62
|
+
multipart-post (>= 1.2, < 3)
|
63
|
+
faraday_middleware (0.10.0)
|
64
|
+
faraday (>= 0.7.4, < 0.10)
|
47
65
|
gherkin (3.2.0)
|
66
|
+
hashie (3.4.3)
|
48
67
|
i18n (0.7.0)
|
49
68
|
iruby (0.2.7)
|
50
69
|
bond (~> 0.5)
|
@@ -56,6 +75,7 @@ GEM
|
|
56
75
|
minitest (5.8.4)
|
57
76
|
multi_json (1.11.2)
|
58
77
|
multi_test (0.1.2)
|
78
|
+
multipart-post (2.0.0)
|
59
79
|
net-sftp (2.1.2)
|
60
80
|
net-ssh (>= 2.6.5)
|
61
81
|
net-ssh (3.1.1)
|
@@ -77,6 +97,11 @@ GEM
|
|
77
97
|
prawn (~> 0.8.4)
|
78
98
|
prawn-svg (~> 0.9.1)
|
79
99
|
text-table (~> 1.2)
|
100
|
+
restforce (2.2.0)
|
101
|
+
faraday (~> 0.9.0)
|
102
|
+
faraday_middleware (>= 0.8.8)
|
103
|
+
hashie (>= 1.2.0, < 4.0)
|
104
|
+
json (>= 1.7.5, < 1.9.0)
|
80
105
|
rspec (3.4.0)
|
81
106
|
rspec-core (~> 3.4.0)
|
82
107
|
rspec-expectations (~> 3.4.0)
|
@@ -97,13 +122,17 @@ GEM
|
|
97
122
|
thread_safe (0.3.5)
|
98
123
|
tzinfo (1.2.2)
|
99
124
|
thread_safe (~> 0.1)
|
125
|
+
xml-simple (1.1.5)
|
100
126
|
|
101
127
|
PLATFORMS
|
102
128
|
ruby
|
103
129
|
|
104
130
|
DEPENDENCIES
|
131
|
+
daru (= 0.1.2)!
|
105
132
|
iruby (= 0.2.7)
|
106
133
|
remi!
|
134
|
+
restforce (~> 2.1)
|
135
|
+
salesforce_bulk_api!
|
107
136
|
|
108
137
|
BUNDLED WITH
|
109
138
|
1.11.2
|
@@ -0,0 +1,17 @@
|
|
1
|
+
Feature: This tests the application of metadata.
|
2
|
+
We need some additional tests to check for errors in parsing.
|
3
|
+
|
4
|
+
|
5
|
+
Background:
|
6
|
+
Given the job is 'Metadata'
|
7
|
+
And the job source 'Source Data'
|
8
|
+
And the job target 'Target Data'
|
9
|
+
And the source 'Source Data'
|
10
|
+
And the target 'Target Data'
|
11
|
+
|
12
|
+
And the following example record for 'Source Data':
|
13
|
+
| activity_id | student_id | student_dob | activity_type | activity_counter | activity_score | activity_cost | activity_date | source_filename |
|
14
|
+
| 1 | 1 | 3/3/1998 | A | 1 | 3.8 | 12.23 | 1/3/2016 03:22:36 | one.csv |
|
15
|
+
|
16
|
+
Scenario: Metadata is used to parse date fields
|
17
|
+
Then the target field 'student_dob' is set to the value "1998-03-03"
|
@@ -110,7 +110,7 @@ Given /^the (source|target) file contains at least the following headers in no p
|
|
110
110
|
field = row.first
|
111
111
|
step "the #{st} field '#{field}'"
|
112
112
|
end
|
113
|
-
expect(@brt.send(st.to_sym).
|
113
|
+
expect(@brt.send(st.to_sym).data_subject.df.vectors.to_a).to include(*@brt.send(st.to_sym).fields.field_names)
|
114
114
|
end
|
115
115
|
|
116
116
|
Given /^the (source|target) file contains all of the following headers in this order:$/ do |st, table|
|
@@ -120,7 +120,7 @@ Given /^the (source|target) file contains all of the following headers in this o
|
|
120
120
|
end
|
121
121
|
|
122
122
|
@brt.run_transforms if st == 'target'
|
123
|
-
expect(@brt.send(st.to_sym).
|
123
|
+
expect(@brt.send(st.to_sym).data_subject.df.vectors.to_a).to eq @brt.send(st.to_sym).fields.field_names
|
124
124
|
end
|
125
125
|
|
126
126
|
|
@@ -188,7 +188,7 @@ Given /^the source field '([^']+)' is parsed with the date format "([^"]*)"$/ do
|
|
188
188
|
step "the source field '#{source_field}'"
|
189
189
|
|
190
190
|
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
191
|
-
expect(@brt.sources[source_name].fields[source_field_name].metadata[:
|
191
|
+
expect(@brt.sources[source_name].fields[source_field_name].metadata[:in_format]).to eq date_format
|
192
192
|
end
|
193
193
|
|
194
194
|
Given /^the source field is parsed with the date format "([^"]*)"$/ do |date_format|
|
@@ -452,8 +452,8 @@ Then /^the target field '([^']+)' is populated from the source field '([^']+)' u
|
|
452
452
|
source_name, source_field_name = @brt.sources.parse_full_field(source_field)
|
453
453
|
target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
|
454
454
|
|
455
|
-
source_format = @brt.sources[source_name].fields[source_field_name].metadata[:
|
456
|
-
source_reformatted = Remi::Transform
|
455
|
+
source_format = @brt.sources[source_name].fields[source_field_name].metadata[:in_format]
|
456
|
+
source_reformatted = Remi::Transform::FormatDate.new(in_format: source_format, out_format: target_format).to_proc
|
457
457
|
.call(@brt.sources[source_name].fields[source_field_name].value)
|
458
458
|
|
459
459
|
@brt.run_transforms
|
@@ -592,7 +592,7 @@ Then /^only the following fields should be present on the target:$/ do |table|
|
|
592
592
|
end
|
593
593
|
|
594
594
|
@brt.run_transforms
|
595
|
-
expect(@brt.target.
|
595
|
+
expect(@brt.target.data_subject.df.vectors.to_a).to match_array @brt.target.fields.field_names
|
596
596
|
end
|
597
597
|
|
598
598
|
### Record-level expectations
|
@@ -8,6 +8,7 @@ Feature: Tests the date_diff transform
|
|
8
8
|
And the source 'Source Data'
|
9
9
|
And the target 'Target Data'
|
10
10
|
|
11
|
+
|
11
12
|
Scenario Outline: Calculating date difference in days2.
|
12
13
|
Given the job parameter 'measure' is "days"
|
13
14
|
And the source field 'Date1' has the value "<Date1>"
|
data/jobs/aggregate_job.rb
CHANGED
data/jobs/all_jobs_shared.rb
CHANGED
data/jobs/copy_source_job.rb
CHANGED
data/jobs/csv_file_target_job.rb
CHANGED
@@ -0,0 +1,60 @@
|
|
1
|
+
require_relative 'all_jobs_shared'
|
2
|
+
|
3
|
+
class MetadataJob
|
4
|
+
include AllJobsShared
|
5
|
+
|
6
|
+
define_source :source_data, Remi::DataSource::DataFrame,
|
7
|
+
fields: {
|
8
|
+
:activity_id => { from: 'in', in: true, cdc_type: 2 },
|
9
|
+
:student_id => { from: 'in', in: true, type: :string, cdc_type: 2 },
|
10
|
+
:student_dob => { from: 'in', in: true, type: :date, in_format: '%m/%d/%Y', out_format: '%Y-%m-%d', cdc_type: 2 },
|
11
|
+
:activity_type => { from: 'in', in: true, type: :string, valid_values: ['A', 'B', 'C'], cdc_type: 2 },
|
12
|
+
:activity_counter => { from: 'in', in: true, type: :integer, cdc_type: 2 },
|
13
|
+
:activity_score => { from: 'in', in: true, type: :float, cdc_type: 2 },
|
14
|
+
:activity_cost => { from: 'in', in: true, type: :decimal, precision: 16, scale: 2, cdc_type: 2 },
|
15
|
+
:activity_date => { from: 'in', in: true, type: :datetime, in_format: '%m/%d/%Y %H:%M:%S', out_format: '%Y-%m-%dT%H:%M:%S', cdc_type: 2 },
|
16
|
+
:source_filename => { from: 'in', in: true, type: :string, cdc_type: 1 }
|
17
|
+
}
|
18
|
+
|
19
|
+
define_target :target_data, Remi::DataTarget::CsvFile,
|
20
|
+
path: "#{Remi::Settings.work_dir}/target_data.csv",
|
21
|
+
fields: {
|
22
|
+
:activity_id => { from: 'out', out: true },
|
23
|
+
:student_id => { from: 'out', out: true, type: :string },
|
24
|
+
:student_dob => { from: 'out', out: true, type: :date, in_format: '%m/%d/%Y', out_format: '%Y-%m-%d' },
|
25
|
+
:activity_type => { from: 'out', out: true, type: :string, valid_values: ['A', 'B', 'C'] },
|
26
|
+
:activity_counter => { from: 'out', out: true, type: :integer },
|
27
|
+
:activity_score => { from: 'out', out: true, type: :float },
|
28
|
+
:activity_cost => { from: 'out', out: true, type: :decimal, precision: 16, scale: 2 },
|
29
|
+
:activity_date => { from: 'out', out: true, type: :datetime, in_format: '%m/%d/%Y %H:%M:%S', out_format: '%Y-%m-%dT%H:%M:%S' },
|
30
|
+
:source_filename => { from: 'out', out: true, type: :string, cdc_type: 1 }
|
31
|
+
}
|
32
|
+
|
33
|
+
define_transform :main do
|
34
|
+
|
35
|
+
=begin
|
36
|
+
source_data.df = Remi::DataFrame.daru([
|
37
|
+
['1','1','3/3/1998','A','1','3.8','12.23','1/3/2016 03:22:36','one.csv'],
|
38
|
+
['2','1','3/3/1998','B','3','4.2','10.53','1/3/2016 03:58:22','one.csv'],
|
39
|
+
['2','1','','B','2','4.23','10.539','1/3/2016 03:58:22','one.csv']
|
40
|
+
].transpose, order: [
|
41
|
+
:activity_id,
|
42
|
+
:student_id,
|
43
|
+
:student_dob,
|
44
|
+
:activity_type,
|
45
|
+
:activity_counter,
|
46
|
+
:activity_score,
|
47
|
+
:activity_cost,
|
48
|
+
:activity_date,
|
49
|
+
:source_filename
|
50
|
+
])
|
51
|
+
=end
|
52
|
+
|
53
|
+
Remi::SourceToTargetMap.apply(source_data.df, target_data.df, source_metadata: source_data.fields) do
|
54
|
+
target_data.fields.keys.each do |field|
|
55
|
+
map source(field) .target(field)
|
56
|
+
.transform(Remi::Transform::EnforceType.new)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
data/jobs/parameters_job.rb
CHANGED
@@ -14,7 +14,7 @@ class ParametersJob
|
|
14
14
|
define_transform :main do
|
15
15
|
Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
|
16
16
|
map source(nil) .target(:myparam)
|
17
|
-
.transform(Remi::Transform
|
17
|
+
.transform(Remi::Transform::Constant.new(params[:myparam]))
|
18
18
|
map source(:parameter_name) .target(:parameter_name)
|
19
19
|
.transform(->(v) { params[v.to_sym] })
|
20
20
|
end
|
data/jobs/sample_job.rb
CHANGED
@@ -1,10 +1,9 @@
|
|
1
1
|
# This is an example Remi job that was auto-generated by Remi.
|
2
|
-
|
3
2
|
require_relative 'all_jobs_shared'
|
3
|
+
require 'remi/data_subject/salesforce'
|
4
4
|
|
5
5
|
class SampleJob
|
6
6
|
include AllJobsShared
|
7
|
-
using Remi::Refinements::Daru
|
8
7
|
|
9
8
|
define_source :existing_contacts, Remi::DataSource::Salesforce,
|
10
9
|
object: :Contact,
|
@@ -14,7 +13,7 @@ class SampleJob
|
|
14
13
|
:Id => {},
|
15
14
|
:External_ID__c => {},
|
16
15
|
:IsActive => { type: :boolean },
|
17
|
-
:CreatedDate => { type: :date,
|
16
|
+
:CreatedDate => { type: :date, in_format: '%Y-%m-%d %H:%M:%S' }
|
18
17
|
},
|
19
18
|
query: <<-EOQ
|
20
19
|
SELECT
|
@@ -50,8 +49,8 @@ class SampleJob
|
|
50
49
|
:mailing_city => {},
|
51
50
|
:mailing_state => {},
|
52
51
|
:mailing_postal_code => {},
|
53
|
-
:birthdate => { type: :date,
|
54
|
-
:applied_date => { type: :date,
|
52
|
+
:birthdate => { type: :date, in_format: '%m/%d/%Y'},
|
53
|
+
:applied_date => { type: :date, in_format: '%m/%d/%Y'}
|
55
54
|
}
|
56
55
|
|
57
56
|
define_target :all_contacts, Remi::DataTarget::DataFrame
|
@@ -83,7 +82,7 @@ class SampleJob
|
|
83
82
|
all_contacts.df = sample_file.df.dup
|
84
83
|
Remi::SourceToTargetMap.apply(all_contacts.df) do
|
85
84
|
map source(:program) .target(:Major__c)
|
86
|
-
.transform(Remi::Transform
|
85
|
+
.transform(Remi::Transform::Lookup.new(params[:program_name_lookup]))
|
87
86
|
end
|
88
87
|
all_contacts.df = all_contacts.df.where(all_contacts.df[:Major__c].not_eq(nil))
|
89
88
|
|
@@ -94,8 +93,8 @@ class SampleJob
|
|
94
93
|
|
95
94
|
# Prefixes source id record and then looks up existing salesforce Id
|
96
95
|
map source(:student_id) .target(:External_ID__c, :Id)
|
97
|
-
.transform(Remi::Transform
|
98
|
-
.transform(->(v) { [v, Remi::Transform
|
96
|
+
.transform(Remi::Transform::Prefix.new('SAMP'))
|
97
|
+
.transform(->(v) { [v, Remi::Transform::Lookup.new(student_id_to_sf_id).call(v)] })
|
99
98
|
end
|
100
99
|
end
|
101
100
|
|
@@ -108,36 +107,36 @@ class SampleJob
|
|
108
107
|
map source(:school_id) .target(:School_ID__c)
|
109
108
|
map source(:school_name) .target(:School_Name__c)
|
110
109
|
map source(:first_name) .target(:FirstName)
|
111
|
-
.transform(Remi::Transform
|
110
|
+
.transform(Remi::Transform::IfBlank.new('Not Provided'))
|
112
111
|
map source(:last_name) .target(:LastName)
|
113
|
-
.transform(Remi::Transform
|
112
|
+
.transform(Remi::Transform::IfBlank.new('Not Provided'))
|
114
113
|
map source(:mailing_city) .target(:MailingCity)
|
115
114
|
map source(:mailing_state) .target(:MailingState)
|
116
115
|
map source(:mailing_postal_code) .target(:MailingPostalCode)
|
117
116
|
|
118
117
|
map source(:birthdate) .target(:Birthdate)
|
119
|
-
.transform(Remi::Transform
|
118
|
+
.transform(Remi::Transform::FormatDate.new(in_format: sample_file.fields[:birthdate][:in_format]))
|
120
119
|
|
121
120
|
map source(:applied_date) .target(:Applied_Date__c)
|
122
|
-
.transform(Remi::Transform
|
123
|
-
.transform(Remi::Transform
|
121
|
+
.transform(Remi::Transform::IfBlank.new(Date.today.strftime(sample_file.fields[:applied_date][:in_format])))
|
122
|
+
.transform(Remi::Transform::FormatDate.new(in_format: sample_file.fields[:applied_date][:in_format]))
|
124
123
|
|
125
124
|
map source(:mailing_address_line_1, :mailing_address_line_2) .target(:MailingStreet)
|
126
125
|
.transform(->(line_1, line_2) {
|
127
|
-
Remi::Transform
|
126
|
+
Remi::Transform::IfBlank.new(nil).call(line_1).nil? ? [] : [line_1, line_2]
|
128
127
|
})
|
129
|
-
.transform(Remi::Transform
|
128
|
+
.transform(Remi::Transform::Concatenate.new(', '))
|
130
129
|
|
131
130
|
map source(:school_id, :school_name) .target(:School__c)
|
132
131
|
.transform(->(id, name) {[
|
133
|
-
Remi::Transform
|
134
|
-
Remi::Transform
|
132
|
+
Remi::Transform::IfBlank.new("Unknown").call(id),
|
133
|
+
Remi::Transform::IfBlank.new("Unknown").call(name)
|
135
134
|
]})
|
136
|
-
.transform(Remi::Transform
|
135
|
+
.transform(Remi::Transform::Concatenate.new('-'))
|
137
136
|
|
138
137
|
map source(:current_email) .target(:Email)
|
139
|
-
.transform(Remi::Transform
|
140
|
-
.transform(Remi::Transform
|
138
|
+
.transform(Remi::Transform::Replace.new(/,/, '.'))
|
139
|
+
.transform(Remi::Transform::ValidateEmail.new)
|
141
140
|
end
|
142
141
|
|
143
142
|
contact_creates.df = work_contact_creates[
|
@@ -15,7 +15,7 @@ class DateDiffJob
|
|
15
15
|
Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
|
16
16
|
map source(:date1, :date2) .target(:difference)
|
17
17
|
.transform(->(d1,d2) { [Date.strptime(d1), Date.strptime(d2)] })
|
18
|
-
.transform(Remi::Transform
|
18
|
+
.transform(Remi::Transform::DateDiff.new(params[:measure]))
|
19
19
|
end
|
20
20
|
end
|
21
21
|
end
|
data/jobs/transforms/nvl_job.rb
CHANGED
@@ -15,7 +15,7 @@ class NvlJob
|
|
15
15
|
define_transform :main, sources: :source_data, targets: :target_data do
|
16
16
|
Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
|
17
17
|
map source(:field1, :field2, :field3) .target(:result_field)
|
18
|
-
.transform(Remi::Transform
|
18
|
+
.transform(Remi::Transform::Nvl.new(params[:default]))
|
19
19
|
end
|
20
20
|
end
|
21
21
|
end
|
@@ -7,19 +7,22 @@ class ParseDateJob
|
|
7
7
|
define_param :if_blank, nil
|
8
8
|
define_source :source_data, Remi::DataSource::DataFrame,
|
9
9
|
fields: {
|
10
|
-
:date_string => { type: :date,
|
11
|
-
:stubbed_date => { type: :date,
|
10
|
+
:date_string => { type: :date, in_format: params[:format] },
|
11
|
+
:stubbed_date => { type: :date, in_format: params[:format] }
|
12
12
|
}
|
13
13
|
define_target :target_data, Remi::DataTarget::DataFrame
|
14
14
|
|
15
15
|
define_transform :main, sources: :source_data, targets: :target_data do
|
16
16
|
|
17
|
+
# Only needed for testing, would be nice to make it testable without this
|
18
|
+
params[:if_blank] = ['high', 'low'].include?(params[:if_blank]) ? params[:if_blank].to_sym : params[:if_blank]
|
19
|
+
|
17
20
|
Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
|
18
21
|
map source(:date_string) .target(:parsed_date)
|
19
|
-
.transform(Remi::Transform
|
22
|
+
.transform(Remi::Transform::ParseDate.new(in_format: params[:format], if_blank: params[:if_blank]))
|
20
23
|
|
21
24
|
map source(:stubbed_date) .target(:parsed_stubbed_date)
|
22
|
-
.transform(Remi::Transform
|
25
|
+
.transform(Remi::Transform::ParseDate.new(in_format: params[:format], if_blank: params[:if_blank]))
|
23
26
|
end
|
24
27
|
end
|
25
28
|
end
|
@@ -13,7 +13,7 @@ class PrefixJob
|
|
13
13
|
define_transform :main, sources: :source_data, targets: :target_data do
|
14
14
|
Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
|
15
15
|
map source(:my_field) .target(:prefixed_field)
|
16
|
-
.transform(Remi::Transform
|
16
|
+
.transform(Remi::Transform::Prefix.new(params[:prefix]))
|
17
17
|
end
|
18
18
|
end
|
19
19
|
end
|