remi 0.2.42 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.yardopts +7 -0
- data/Gemfile +1 -1
- data/Gemfile.lock +13 -26
- data/README.md +1 -1
- data/features/step_definitions/remi_step.rb +33 -13
- data/features/sub_job_example.feature +24 -0
- data/features/sub_transform_example.feature +35 -0
- data/features/sub_transform_many_to_many.feature +49 -0
- data/features/support/env_app.rb +1 -1
- data/jobs/all_jobs_shared.rb +19 -16
- data/jobs/copy_source_job.rb +11 -9
- data/jobs/csv_file_target_job.rb +10 -9
- data/jobs/json_job.rb +18 -14
- data/jobs/metadata_job.rb +33 -28
- data/jobs/parameters_job.rb +14 -11
- data/jobs/sample_job.rb +106 -77
- data/jobs/sftp_file_target_job.rb +14 -13
- data/jobs/sub_job_example_job.rb +86 -0
- data/jobs/sub_transform_example_job.rb +43 -0
- data/jobs/sub_transform_many_to_many_job.rb +46 -0
- data/jobs/transforms/concatenate_job.rb +16 -12
- data/jobs/transforms/data_frame_sieve_job.rb +24 -19
- data/jobs/transforms/date_diff_job.rb +15 -11
- data/jobs/transforms/nvl_job.rb +16 -12
- data/jobs/transforms/parse_date_job.rb +17 -14
- data/jobs/transforms/partitioner_job.rb +27 -19
- data/jobs/transforms/prefix_job.rb +13 -10
- data/jobs/transforms/truncate_job.rb +14 -10
- data/jobs/transforms/truthy_job.rb +11 -8
- data/lib/remi.rb +25 -11
- data/lib/remi/data_frame.rb +4 -4
- data/lib/remi/data_frame/daru.rb +1 -37
- data/lib/remi/data_subject.rb +234 -48
- data/lib/remi/data_subjects/csv_file.rb +171 -0
- data/lib/remi/data_subjects/data_frame.rb +106 -0
- data/lib/remi/data_subjects/file_system.rb +115 -0
- data/lib/remi/data_subjects/local_file.rb +109 -0
- data/lib/remi/data_subjects/none.rb +31 -0
- data/lib/remi/data_subjects/postgres.rb +186 -0
- data/lib/remi/data_subjects/s3_file.rb +84 -0
- data/lib/remi/data_subjects/salesforce.rb +211 -0
- data/lib/remi/data_subjects/sftp_file.rb +196 -0
- data/lib/remi/data_subjects/sub_job.rb +50 -0
- data/lib/remi/dsl.rb +74 -0
- data/lib/remi/encoder.rb +45 -0
- data/lib/remi/extractor.rb +21 -0
- data/lib/remi/field_symbolizers.rb +1 -0
- data/lib/remi/job.rb +279 -113
- data/lib/remi/job/parameters.rb +90 -0
- data/lib/remi/job/sub_job.rb +35 -0
- data/lib/remi/job/transform.rb +165 -0
- data/lib/remi/loader.rb +22 -0
- data/lib/remi/monkeys/daru.rb +4 -0
- data/lib/remi/parser.rb +44 -0
- data/lib/remi/testing/business_rules.rb +17 -23
- data/lib/remi/testing/data_stub.rb +2 -2
- data/lib/remi/version.rb +1 -1
- data/remi.gemspec +3 -0
- data/spec/data_subject_spec.rb +475 -11
- data/spec/data_subjects/csv_file_spec.rb +69 -0
- data/spec/data_subjects/data_frame_spec.rb +52 -0
- data/spec/{extractor → data_subjects}/file_system_spec.rb +0 -0
- data/spec/{extractor → data_subjects}/local_file_spec.rb +0 -0
- data/spec/data_subjects/none_spec.rb +41 -0
- data/spec/data_subjects/postgres_spec.rb +80 -0
- data/spec/{extractor → data_subjects}/s3_file_spec.rb +0 -0
- data/spec/data_subjects/salesforce_spec.rb +117 -0
- data/spec/{extractor → data_subjects}/sftp_file_spec.rb +16 -0
- data/spec/data_subjects/sub_job_spec.rb +33 -0
- data/spec/encoder_spec.rb +38 -0
- data/spec/extractor_spec.rb +11 -0
- data/spec/fixtures/sf_bulk_helper_stubs.rb +443 -0
- data/spec/job/transform_spec.rb +257 -0
- data/spec/job_spec.rb +507 -0
- data/spec/loader_spec.rb +11 -0
- data/spec/parser_spec.rb +38 -0
- data/spec/sf_bulk_helper_spec.rb +117 -0
- data/spec/testing/data_stub_spec.rb +5 -3
- metadata +109 -27
- data/features/aggregate.feature +0 -42
- data/jobs/aggregate_job.rb +0 -31
- data/jobs/transforms/transform_jobs.rb +0 -4
- data/lib/remi/data_subject/csv_file.rb +0 -162
- data/lib/remi/data_subject/data_frame.rb +0 -52
- data/lib/remi/data_subject/postgres.rb +0 -134
- data/lib/remi/data_subject/salesforce.rb +0 -136
- data/lib/remi/data_subject/sftp_file.rb +0 -65
- data/lib/remi/extractor/file_system.rb +0 -92
- data/lib/remi/extractor/local_file.rb +0 -43
- data/lib/remi/extractor/s3_file.rb +0 -57
- data/lib/remi/extractor/sftp_file.rb +0 -83
- data/spec/data_subject/csv_file_spec.rb +0 -79
- data/spec/data_subject/data_frame.rb +0 -27
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 71164673ece850e218a1ef18a72aa02f4ca5d065
|
4
|
+
data.tar.gz: b52e6e86bc4995f2caf79dad79a7f0a195cdb9f9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 708b6fdf566acb525caa85bc038685bad46c3999f65045b5281c9393d731f5a3cdb49ac30a7ecde12f16abab5a27364449a83fc840adfa531e2a04a783fcd69a
|
7
|
+
data.tar.gz: 48730dd300bb84dadb93d911e37baa274958d0d1deaddcbc769aa43a7f7c1887ffbcf025ce43d4188e09630de31868acc6315ae2559dbd80853febc42e13b5d7
|
data/.yardopts
ADDED
data/Gemfile
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
source 'https://rubygems.org'
|
3
3
|
|
4
4
|
gemspec
|
5
|
-
gem 'daru', '0.1.
|
5
|
+
gem 'daru', '0.1.4.1', git: 'git@github.com:inside-track/daru.git', branch: '0.1.4.1-Remi'
|
6
6
|
gem 'restforce', '~> 2.1'
|
7
7
|
gem 'salesforce_bulk_api', git: 'git@github.com:inside-track/salesforce_bulk_api.git', branch: 'master'
|
8
8
|
gem 'aws-sdk', '~> 2.3'
|
data/Gemfile.lock
CHANGED
@@ -1,11 +1,10 @@
|
|
1
1
|
GIT
|
2
2
|
remote: git@github.com:inside-track/daru.git
|
3
|
-
revision:
|
4
|
-
branch:
|
3
|
+
revision: e38e98890c99dee03ceedaca3a5faf40859042ab
|
4
|
+
branch: 0.1.4.1-Remi
|
5
5
|
specs:
|
6
|
-
daru (0.1.
|
7
|
-
|
8
|
-
spreadsheet (~> 1.1.1)
|
6
|
+
daru (0.1.4.1)
|
7
|
+
backports
|
9
8
|
|
10
9
|
GIT
|
11
10
|
remote: git@github.com:inside-track/salesforce_bulk_api.git
|
@@ -19,7 +18,7 @@ GIT
|
|
19
18
|
PATH
|
20
19
|
remote: .
|
21
20
|
specs:
|
22
|
-
remi (0.
|
21
|
+
remi (0.3.0)
|
23
22
|
activesupport (~> 4.2)
|
24
23
|
bond (~> 0.5)
|
25
24
|
cucumber (~> 2.1)
|
@@ -46,9 +45,9 @@ GEM
|
|
46
45
|
jmespath (~> 1.0)
|
47
46
|
aws-sdk-resources (2.3.5)
|
48
47
|
aws-sdk-core (= 2.3.5)
|
48
|
+
backports (3.6.8)
|
49
49
|
bond (0.5.1)
|
50
50
|
builder (3.2.2)
|
51
|
-
clbustos-rtf (0.4.2)
|
52
51
|
cucumber (2.3.3)
|
53
52
|
builder (>= 2.1.2)
|
54
53
|
cucumber-core (~> 1.4.0)
|
@@ -69,6 +68,7 @@ GEM
|
|
69
68
|
faraday_middleware (0.10.0)
|
70
69
|
faraday (>= 0.7.4, < 0.10)
|
71
70
|
gherkin (3.2.0)
|
71
|
+
github-markup (1.4.0)
|
72
72
|
hashie (3.4.3)
|
73
73
|
i18n (0.7.0)
|
74
74
|
iruby (0.2.7)
|
@@ -89,23 +89,10 @@ GEM
|
|
89
89
|
net-ssh (>= 2.6.5)
|
90
90
|
net-ssh (3.1.1)
|
91
91
|
pg (0.18.4)
|
92
|
-
prawn (0.8.4)
|
93
|
-
prawn-core (>= 0.8.4, < 0.9)
|
94
|
-
prawn-layout (>= 0.8.4, < 0.9)
|
95
|
-
prawn-security (>= 0.8.4, < 0.9)
|
96
|
-
prawn-core (0.8.4)
|
97
|
-
prawn-layout (0.8.4)
|
98
|
-
prawn-security (0.8.4)
|
99
|
-
prawn-svg (0.9.1.11)
|
100
|
-
prawn (>= 0.8.4)
|
101
92
|
rbczmq (1.7.9)
|
93
|
+
redcarpet (3.3.4)
|
102
94
|
regex_sieve (0.1.0)
|
103
95
|
regexp-examples (1.2.0)
|
104
|
-
reportbuilder (1.4.2)
|
105
|
-
clbustos-rtf (~> 0.4.0)
|
106
|
-
prawn (~> 0.8.4)
|
107
|
-
prawn-svg (~> 0.9.1)
|
108
|
-
text-table (~> 1.2)
|
109
96
|
restforce (2.2.0)
|
110
97
|
faraday (~> 0.9.0)
|
111
98
|
faraday_middleware (>= 0.8.8)
|
@@ -124,25 +111,25 @@ GEM
|
|
124
111
|
diff-lcs (>= 1.2.0, < 2.0)
|
125
112
|
rspec-support (~> 3.4.0)
|
126
113
|
rspec-support (3.4.1)
|
127
|
-
ruby-ole (1.2.12)
|
128
|
-
spreadsheet (1.1.2)
|
129
|
-
ruby-ole (>= 1.0)
|
130
|
-
text-table (1.2.4)
|
131
114
|
thread_safe (0.3.5)
|
132
115
|
tzinfo (1.2.2)
|
133
116
|
thread_safe (~> 0.1)
|
134
117
|
xml-simple (1.1.5)
|
118
|
+
yard (0.9.0)
|
135
119
|
|
136
120
|
PLATFORMS
|
137
121
|
ruby
|
138
122
|
|
139
123
|
DEPENDENCIES
|
140
124
|
aws-sdk (~> 2.3)
|
141
|
-
daru (= 0.1.
|
125
|
+
daru (= 0.1.4.1)!
|
126
|
+
github-markup
|
142
127
|
iruby (= 0.2.7)
|
128
|
+
redcarpet
|
143
129
|
remi!
|
144
130
|
restforce (~> 2.1)
|
145
131
|
salesforce_bulk_api!
|
132
|
+
yard
|
146
133
|
|
147
134
|
BUNDLED WITH
|
148
135
|
1.11.2
|
data/README.md
CHANGED
@@ -44,57 +44,65 @@ end
|
|
44
44
|
### Source file processing
|
45
45
|
|
46
46
|
Given /^files with names matching the pattern \/(.*)\/$/ do |pattern|
|
47
|
-
expect(@brt.source.data_subject.
|
47
|
+
expect(@brt.source.data_subject.extractors.map(&:pattern)).to include Regexp.new(pattern)
|
48
48
|
end
|
49
49
|
|
50
50
|
Given /^download groups defined by the pattern \/(.*)\/$/ do |pattern|
|
51
|
-
expect(@brt.source.data_subject.
|
51
|
+
expect(@brt.source.data_subject.extractors.map(&:group_by)).to include Regexp.new(pattern)
|
52
52
|
end
|
53
53
|
|
54
54
|
Then /^the file with the latest date stamp will be downloaded for processing$/ do
|
55
|
-
expect(@brt.source.data_subject.
|
55
|
+
expect(@brt.source.data_subject.extractors.map(&:most_recent_by)).to include :create_time
|
56
56
|
end
|
57
57
|
|
58
58
|
Then /^all files matching the pattern will be downloaded for processing$/ do
|
59
|
-
expect(@brt.source.data_subject.
|
59
|
+
expect(@brt.source.data_subject.extractors.map(&:most_recent_only)).not_to include eq true
|
60
60
|
end
|
61
61
|
|
62
62
|
Then /^the file that comes last in an alphanumeric sort by group will be downloaded for processing$/ do
|
63
|
-
expect(@brt.source.data_subject.
|
63
|
+
expect(@brt.source.data_subject.extractors.map(&:most_recent_by)).to include :name
|
64
64
|
end
|
65
65
|
|
66
66
|
Then /^the file is uploaded to the remote path "([^"]+)"$/ do |remote_path|
|
67
|
-
|
67
|
+
expected_path = Remi::Testing::BusinessRules::ParseFormula.parse(remote_path)
|
68
|
+
expect(@brt.target.data_subject.loaders.map(&:remote_path)).to include expected_path
|
68
69
|
end
|
69
70
|
|
70
71
|
## CSV Options
|
71
72
|
|
72
73
|
Given /^the (source|target) file is delimited with a (\w+)$/ do |st, delimiter|
|
73
|
-
|
74
|
+
csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
|
75
|
+
expect(csv_subject.csv_options[:col_sep]).to eq Remi::Testing::BusinessRules.csv_opt_map[delimiter]
|
74
76
|
end
|
75
77
|
|
76
78
|
Given /^the (source|target) file is encoded using "([^"]+)" format$/ do |st, encoding|
|
77
|
-
|
79
|
+
csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
|
80
|
+
expect(csv_subject.csv_options[:encoding].split(':').first).to eq encoding
|
78
81
|
end
|
79
82
|
|
80
83
|
Given /^the (source|target) file uses a ([\w ]+) to quote embedded delimiters$/ do |st, quote_char|
|
81
|
-
|
84
|
+
csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
|
85
|
+
expect(csv_subject.csv_options[:quote_char]).to eq Remi::Testing::BusinessRules.csv_opt_map[quote_char]
|
82
86
|
end
|
83
87
|
|
84
88
|
Given /^the (source|target) file uses a preceding ([\w ]+) to escape an embedded quoting character$/ do |st, escape_char|
|
85
|
-
|
89
|
+
csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
|
90
|
+
expect(csv_subject.csv_options[:quote_char]).to eq Remi::Testing::BusinessRules.csv_opt_map[escape_char]
|
86
91
|
end
|
87
92
|
|
88
93
|
Given /^the (source|target) file uses ([\w ]+) line endings$/ do |st, line_endings|
|
89
|
-
|
94
|
+
csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
|
95
|
+
expect(csv_subject.csv_options[:row_sep]).to eq Remi::Testing::BusinessRules.csv_opt_map[line_endings]
|
90
96
|
end
|
91
97
|
|
92
98
|
Given /^the (source|target) file uses "([^"]+)" as a record separator$/ do |st, line_endings|
|
93
|
-
|
99
|
+
csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
|
100
|
+
expect(csv_subject.csv_options[:row_sep]).to eq line_endings.gsub(/\\n/, "\n").gsub(/\\r/, "\r")
|
94
101
|
end
|
95
102
|
|
96
103
|
Given /^the (source|target) file (contains|does not contain) a header row$/ do |st, header|
|
97
|
-
|
104
|
+
csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
|
105
|
+
expect(csv_subject.csv_options[:headers]).to eq (header == 'contains')
|
98
106
|
end
|
99
107
|
|
100
108
|
Given /^the (source|target) file contains at least the following headers in no particular order:$/ do |st, table|
|
@@ -301,6 +309,12 @@ Then /^the target '(.+)' should match the example '([[:alnum:]\-\s]+)'$/ do |tar
|
|
301
309
|
target_hash = @brt.targets[target_name].column_hash
|
302
310
|
example_hash = @brt.examples[example_name].column_hash
|
303
311
|
common_keys = target_hash.keys & example_hash.keys
|
312
|
+
expect(common_keys).to match_array(example_hash.keys), <<-EOT
|
313
|
+
Fields in example not found in target
|
314
|
+
Example fields: #{example_hash.keys}
|
315
|
+
Target fields: #{target_hash.keys}
|
316
|
+
Missing fields: #{example_hash.keys - target_hash.keys}
|
317
|
+
EOT
|
304
318
|
|
305
319
|
target_to_compare = target_hash.select { |k,v| common_keys.include? k }
|
306
320
|
target_to_compare.each do |k, v|
|
@@ -317,6 +331,12 @@ Then /^the target should match the example '([[:alnum:]\-\s]+)'$/ do |example_na
|
|
317
331
|
step "the target '#{target_name}' should match the example '#{example_name}'"
|
318
332
|
end
|
319
333
|
|
334
|
+
Then /^the target '(.+)' should match the example:/ do |target_name, example_table|
|
335
|
+
example_name = SecureRandom.uuid
|
336
|
+
@brt.add_example example_name, example_table
|
337
|
+
step "the target '#{target_name}' should match the example '#{example_name}'"
|
338
|
+
end
|
339
|
+
|
320
340
|
Then /^the target should match the example:/ do |example_table|
|
321
341
|
example_name = SecureRandom.uuid
|
322
342
|
@brt.add_example example_name, example_table
|
@@ -0,0 +1,24 @@
|
|
1
|
+
Feature: This is a job that demonstrates how to use Remi subjobs
|
2
|
+
|
3
|
+
Background:
|
4
|
+
Given the job is 'Sub Job Example'
|
5
|
+
And the job target 'Just Beers'
|
6
|
+
And the job target 'Zombified Beers'
|
7
|
+
|
8
|
+
Scenario: Sub jobs can be used to extract data
|
9
|
+
|
10
|
+
Given the target 'Just Beers'
|
11
|
+
|
12
|
+
Then the target should match the example:
|
13
|
+
| Brewer | Style |
|
14
|
+
| Baerlic | IPA |
|
15
|
+
| Ex Novo | Red |
|
16
|
+
|
17
|
+
Scenario: Sub jobs can be used to transform source data
|
18
|
+
|
19
|
+
Given the target 'Zombified Beers'
|
20
|
+
|
21
|
+
Then the target should match the example:
|
22
|
+
| Brewer | Style |
|
23
|
+
| Zombie Baerlic | Zombie IPA |
|
24
|
+
| Zombie Ex Novo | Zombie Red |
|
@@ -0,0 +1,35 @@
|
|
1
|
+
Feature: This is a job that demonstrates how to use Remi sub transforms
|
2
|
+
|
3
|
+
Background:
|
4
|
+
Given the job is 'Sub Transform Example'
|
5
|
+
And the job source 'My Source'
|
6
|
+
And the job target 'My Target'
|
7
|
+
|
8
|
+
Scenario: It uses the subtransform to prefix a field with the default value
|
9
|
+
|
10
|
+
Given the source 'My Source'
|
11
|
+
And the target 'My Target'
|
12
|
+
|
13
|
+
And the following example for 'My Source':
|
14
|
+
| id |
|
15
|
+
| 1 |
|
16
|
+
|
17
|
+
Then the target should match the example:
|
18
|
+
| id | default_id |
|
19
|
+
| 1 | DEFAULT1 |
|
20
|
+
|
21
|
+
|
22
|
+
Scenario: It uses parameters to customize subtansforms
|
23
|
+
|
24
|
+
Given the source 'My Source'
|
25
|
+
And the target 'My Target'
|
26
|
+
|
27
|
+
And the job parameter 'job_prefix' is "UC"
|
28
|
+
|
29
|
+
And the following example for 'My Source':
|
30
|
+
| id |
|
31
|
+
| 1 |
|
32
|
+
|
33
|
+
Then the target should match the example:
|
34
|
+
| id | default_id |
|
35
|
+
| 1 | UC1 |
|
@@ -0,0 +1,49 @@
|
|
1
|
+
Feature: This is a job that demonstrates how to use Remi sub transforms
|
2
|
+
|
3
|
+
Background:
|
4
|
+
Given the job is 'Sub Transform Many To Many'
|
5
|
+
And the job source 'Beer Fact'
|
6
|
+
And the job source 'Beer Dim'
|
7
|
+
And the job target 'Beer Count'
|
8
|
+
And the job target 'Style Count'
|
9
|
+
|
10
|
+
|
11
|
+
Scenario: It runs the subtransform as expected
|
12
|
+
|
13
|
+
Given the source 'Beer Fact'
|
14
|
+
And the source 'Beer Dim'
|
15
|
+
And the target 'Beer Count'
|
16
|
+
And the target 'Style Count'
|
17
|
+
|
18
|
+
And the following example for 'Beer Fact':
|
19
|
+
| fact_sk | beer_sk |
|
20
|
+
| 1 | 1 |
|
21
|
+
| 2 | 1 |
|
22
|
+
| 3 | 1 |
|
23
|
+
| 4 | 2 |
|
24
|
+
| 5 | 2 |
|
25
|
+
| 6 | 2 |
|
26
|
+
| 7 | 3 |
|
27
|
+
| 8 | 3 |
|
28
|
+
| 9 | 3 |
|
29
|
+
| 10 | 4 |
|
30
|
+
|
31
|
+
And the following example for 'Beer Dim':
|
32
|
+
| beer_sk | name | style |
|
33
|
+
| 1 | Invincible | IPA |
|
34
|
+
| 2 | Pipewrench | IPA |
|
35
|
+
| 3 | Altera | Red |
|
36
|
+
| 4 | Urban Farmhouse | Farmhouse |
|
37
|
+
|
38
|
+
Then the target 'Beer Count' should match the example:
|
39
|
+
| name | count |
|
40
|
+
| Altera | 3 |
|
41
|
+
| Invincible | 3 |
|
42
|
+
| Pipewrench | 3 |
|
43
|
+
| Urban Farmhouse | 1 |
|
44
|
+
|
45
|
+
And the target 'Style Count' should match the example:
|
46
|
+
| style | count |
|
47
|
+
| Farmhouse | 1 |
|
48
|
+
| IPA | 6 |
|
49
|
+
| Red | 3 |
|
data/features/support/env_app.rb
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
# Put stuff here that is specific to your testing environment
|
data/jobs/all_jobs_shared.rb
CHANGED
@@ -4,22 +4,25 @@
|
|
4
4
|
require 'bundler/setup'
|
5
5
|
require 'remi'
|
6
6
|
|
7
|
-
|
8
|
-
include Remi::Job
|
7
|
+
class AllJobsShared < Remi::Job
|
9
8
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
9
|
+
param :sftp do
|
10
|
+
{
|
11
|
+
host: 'example.com',
|
12
|
+
username: 'user',
|
13
|
+
password: '1234567890'
|
14
|
+
}
|
15
|
+
end
|
15
16
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
17
|
+
param :salesforce_credentials do
|
18
|
+
{
|
19
|
+
host: 'login.salesforce.com',
|
20
|
+
username: 'user@example.com',
|
21
|
+
password: 'password',
|
22
|
+
security_token: '4342jn3j4n32n4',
|
23
|
+
client_id: 'dkfjsdkfjoasdjdf',
|
24
|
+
client_secret: '28357245723475',
|
25
|
+
instance_url: 'https://na1.salesforce.com'
|
26
|
+
}
|
27
|
+
end
|
25
28
|
end
|
data/jobs/copy_source_job.rb
CHANGED
@@ -1,16 +1,18 @@
|
|
1
1
|
require_relative 'all_jobs_shared'
|
2
2
|
|
3
|
-
class CopySourceJob
|
4
|
-
|
3
|
+
class CopySourceJob < Remi::Job
|
4
|
+
source :source_data do
|
5
|
+
fields(
|
6
|
+
{
|
7
|
+
:some_field => {},
|
8
|
+
:some_date => { type: :date, format: '%Y-%m-%d' }
|
9
|
+
}
|
10
|
+
)
|
11
|
+
end
|
5
12
|
|
6
|
-
|
7
|
-
fields: {
|
8
|
-
:some_field => {},
|
9
|
-
:some_date => { type: :date, format: '%Y-%m-%d' }
|
10
|
-
}
|
11
|
-
define_source :target_data, Remi::DataSource::DataFrame
|
13
|
+
target :target_data
|
12
14
|
|
13
|
-
|
15
|
+
transform :main do
|
14
16
|
target_data.df = source_data.df.dup
|
15
17
|
end
|
16
18
|
end
|
data/jobs/csv_file_target_job.rb
CHANGED
@@ -1,15 +1,16 @@
|
|
1
1
|
require_relative 'all_jobs_shared'
|
2
2
|
|
3
|
-
class CsvFileTargetJob
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
3
|
+
class CsvFileTargetJob < Remi::Job
|
4
|
+
target :some_csv_file do
|
5
|
+
encoder Remi::Encoder::CsvFile.new(
|
6
|
+
path: "#{Remi::Settings.work_dir}/some_file.csv",
|
7
|
+
csv_options: {
|
8
|
+
col_sep: '|'
|
9
|
+
}
|
10
|
+
)
|
11
|
+
end
|
11
12
|
|
12
|
-
|
13
|
+
transform :main do
|
13
14
|
some_csv_file.df = Daru::DataFrame.new({
|
14
15
|
col3: Faker::Hipster.words(10),
|
15
16
|
col1: Faker::Hipster.words(10),
|