remi 0.2.42 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. data/.yardopts +7 -0
  3. data/Gemfile +1 -1
  4. data/Gemfile.lock +13 -26
  5. data/README.md +1 -1
  6. data/features/step_definitions/remi_step.rb +33 -13
  7. data/features/sub_job_example.feature +24 -0
  8. data/features/sub_transform_example.feature +35 -0
  9. data/features/sub_transform_many_to_many.feature +49 -0
  10. data/features/support/env_app.rb +1 -1
  11. data/jobs/all_jobs_shared.rb +19 -16
  12. data/jobs/copy_source_job.rb +11 -9
  13. data/jobs/csv_file_target_job.rb +10 -9
  14. data/jobs/json_job.rb +18 -14
  15. data/jobs/metadata_job.rb +33 -28
  16. data/jobs/parameters_job.rb +14 -11
  17. data/jobs/sample_job.rb +106 -77
  18. data/jobs/sftp_file_target_job.rb +14 -13
  19. data/jobs/sub_job_example_job.rb +86 -0
  20. data/jobs/sub_transform_example_job.rb +43 -0
  21. data/jobs/sub_transform_many_to_many_job.rb +46 -0
  22. data/jobs/transforms/concatenate_job.rb +16 -12
  23. data/jobs/transforms/data_frame_sieve_job.rb +24 -19
  24. data/jobs/transforms/date_diff_job.rb +15 -11
  25. data/jobs/transforms/nvl_job.rb +16 -12
  26. data/jobs/transforms/parse_date_job.rb +17 -14
  27. data/jobs/transforms/partitioner_job.rb +27 -19
  28. data/jobs/transforms/prefix_job.rb +13 -10
  29. data/jobs/transforms/truncate_job.rb +14 -10
  30. data/jobs/transforms/truthy_job.rb +11 -8
  31. data/lib/remi.rb +25 -11
  32. data/lib/remi/data_frame.rb +4 -4
  33. data/lib/remi/data_frame/daru.rb +1 -37
  34. data/lib/remi/data_subject.rb +234 -48
  35. data/lib/remi/data_subjects/csv_file.rb +171 -0
  36. data/lib/remi/data_subjects/data_frame.rb +106 -0
  37. data/lib/remi/data_subjects/file_system.rb +115 -0
  38. data/lib/remi/data_subjects/local_file.rb +109 -0
  39. data/lib/remi/data_subjects/none.rb +31 -0
  40. data/lib/remi/data_subjects/postgres.rb +186 -0
  41. data/lib/remi/data_subjects/s3_file.rb +84 -0
  42. data/lib/remi/data_subjects/salesforce.rb +211 -0
  43. data/lib/remi/data_subjects/sftp_file.rb +196 -0
  44. data/lib/remi/data_subjects/sub_job.rb +50 -0
  45. data/lib/remi/dsl.rb +74 -0
  46. data/lib/remi/encoder.rb +45 -0
  47. data/lib/remi/extractor.rb +21 -0
  48. data/lib/remi/field_symbolizers.rb +1 -0
  49. data/lib/remi/job.rb +279 -113
  50. data/lib/remi/job/parameters.rb +90 -0
  51. data/lib/remi/job/sub_job.rb +35 -0
  52. data/lib/remi/job/transform.rb +165 -0
  53. data/lib/remi/loader.rb +22 -0
  54. data/lib/remi/monkeys/daru.rb +4 -0
  55. data/lib/remi/parser.rb +44 -0
  56. data/lib/remi/testing/business_rules.rb +17 -23
  57. data/lib/remi/testing/data_stub.rb +2 -2
  58. data/lib/remi/version.rb +1 -1
  59. data/remi.gemspec +3 -0
  60. data/spec/data_subject_spec.rb +475 -11
  61. data/spec/data_subjects/csv_file_spec.rb +69 -0
  62. data/spec/data_subjects/data_frame_spec.rb +52 -0
  63. data/spec/{extractor → data_subjects}/file_system_spec.rb +0 -0
  64. data/spec/{extractor → data_subjects}/local_file_spec.rb +0 -0
  65. data/spec/data_subjects/none_spec.rb +41 -0
  66. data/spec/data_subjects/postgres_spec.rb +80 -0
  67. data/spec/{extractor → data_subjects}/s3_file_spec.rb +0 -0
  68. data/spec/data_subjects/salesforce_spec.rb +117 -0
  69. data/spec/{extractor → data_subjects}/sftp_file_spec.rb +16 -0
  70. data/spec/data_subjects/sub_job_spec.rb +33 -0
  71. data/spec/encoder_spec.rb +38 -0
  72. data/spec/extractor_spec.rb +11 -0
  73. data/spec/fixtures/sf_bulk_helper_stubs.rb +443 -0
  74. data/spec/job/transform_spec.rb +257 -0
  75. data/spec/job_spec.rb +507 -0
  76. data/spec/loader_spec.rb +11 -0
  77. data/spec/parser_spec.rb +38 -0
  78. data/spec/sf_bulk_helper_spec.rb +117 -0
  79. data/spec/testing/data_stub_spec.rb +5 -3
  80. metadata +109 -27
  81. data/features/aggregate.feature +0 -42
  82. data/jobs/aggregate_job.rb +0 -31
  83. data/jobs/transforms/transform_jobs.rb +0 -4
  84. data/lib/remi/data_subject/csv_file.rb +0 -162
  85. data/lib/remi/data_subject/data_frame.rb +0 -52
  86. data/lib/remi/data_subject/postgres.rb +0 -134
  87. data/lib/remi/data_subject/salesforce.rb +0 -136
  88. data/lib/remi/data_subject/sftp_file.rb +0 -65
  89. data/lib/remi/extractor/file_system.rb +0 -92
  90. data/lib/remi/extractor/local_file.rb +0 -43
  91. data/lib/remi/extractor/s3_file.rb +0 -57
  92. data/lib/remi/extractor/sftp_file.rb +0 -83
  93. data/spec/data_subject/csv_file_spec.rb +0 -79
  94. data/spec/data_subject/data_frame.rb +0 -27
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 078d9d338ea606fee80194b60e6bd52eeadd1211
4
- data.tar.gz: 05ca9a6b08896ad3266fbfcd61932a71e08c5318
3
+ metadata.gz: 71164673ece850e218a1ef18a72aa02f4ca5d065
4
+ data.tar.gz: b52e6e86bc4995f2caf79dad79a7f0a195cdb9f9
5
5
  SHA512:
6
- metadata.gz: 454633037cc429d0ad056e150ec8f07aa2f064f62bf9ee08e3c85db52f31fd92de59c3dd1d3e3dae27d19163c3146619dd340b751ee70af3e0b98176a3d37611
7
- data.tar.gz: 82a6aef7e5bcece2f2012b2b76f120074adfd3387173d18d1bf4a11aa8d4d6e6994e292da1287a90aab91c01367f7e7dac674111b20c6aaa7b75e4a672e0290c
6
+ metadata.gz: 708b6fdf566acb525caa85bc038685bad46c3999f65045b5281c9393d731f5a3cdb49ac30a7ecde12f16abab5a27364449a83fc840adfa531e2a04a783fcd69a
7
+ data.tar.gz: 48730dd300bb84dadb93d911e37baa274958d0d1deaddcbc769aa43a7f7c1887ffbcf025ce43d4188e09630de31868acc6315ae2559dbd80853febc42e13b5d7
@@ -0,0 +1,7 @@
1
+ --title 'Remi Documentation'
2
+
3
+ --no-private
4
+ --main=README.md
5
+
6
+ --markup-provider=redcarpet
7
+ --markup=markdown
data/Gemfile CHANGED
@@ -2,7 +2,7 @@
2
2
  source 'https://rubygems.org'
3
3
 
4
4
  gemspec
5
- gem 'daru', '0.1.2', git: 'git@github.com:inside-track/daru.git', branch: 'itk-master'
5
+ gem 'daru', '0.1.4.1', git: 'git@github.com:inside-track/daru.git', branch: '0.1.4.1-Remi'
6
6
  gem 'restforce', '~> 2.1'
7
7
  gem 'salesforce_bulk_api', git: 'git@github.com:inside-track/salesforce_bulk_api.git', branch: 'master'
8
8
  gem 'aws-sdk', '~> 2.3'
@@ -1,11 +1,10 @@
1
1
  GIT
2
2
  remote: git@github.com:inside-track/daru.git
3
- revision: 80b8024102bdb31ef402cb8d6c5cef86dd31cd41
4
- branch: itk-master
3
+ revision: e38e98890c99dee03ceedaca3a5faf40859042ab
4
+ branch: 0.1.4.1-Remi
5
5
  specs:
6
- daru (0.1.2)
7
- reportbuilder (~> 1.4)
8
- spreadsheet (~> 1.1.1)
6
+ daru (0.1.4.1)
7
+ backports
9
8
 
10
9
  GIT
11
10
  remote: git@github.com:inside-track/salesforce_bulk_api.git
@@ -19,7 +18,7 @@ GIT
19
18
  PATH
20
19
  remote: .
21
20
  specs:
22
- remi (0.2.42)
21
+ remi (0.3.0)
23
22
  activesupport (~> 4.2)
24
23
  bond (~> 0.5)
25
24
  cucumber (~> 2.1)
@@ -46,9 +45,9 @@ GEM
46
45
  jmespath (~> 1.0)
47
46
  aws-sdk-resources (2.3.5)
48
47
  aws-sdk-core (= 2.3.5)
48
+ backports (3.6.8)
49
49
  bond (0.5.1)
50
50
  builder (3.2.2)
51
- clbustos-rtf (0.4.2)
52
51
  cucumber (2.3.3)
53
52
  builder (>= 2.1.2)
54
53
  cucumber-core (~> 1.4.0)
@@ -69,6 +68,7 @@ GEM
69
68
  faraday_middleware (0.10.0)
70
69
  faraday (>= 0.7.4, < 0.10)
71
70
  gherkin (3.2.0)
71
+ github-markup (1.4.0)
72
72
  hashie (3.4.3)
73
73
  i18n (0.7.0)
74
74
  iruby (0.2.7)
@@ -89,23 +89,10 @@ GEM
89
89
  net-ssh (>= 2.6.5)
90
90
  net-ssh (3.1.1)
91
91
  pg (0.18.4)
92
- prawn (0.8.4)
93
- prawn-core (>= 0.8.4, < 0.9)
94
- prawn-layout (>= 0.8.4, < 0.9)
95
- prawn-security (>= 0.8.4, < 0.9)
96
- prawn-core (0.8.4)
97
- prawn-layout (0.8.4)
98
- prawn-security (0.8.4)
99
- prawn-svg (0.9.1.11)
100
- prawn (>= 0.8.4)
101
92
  rbczmq (1.7.9)
93
+ redcarpet (3.3.4)
102
94
  regex_sieve (0.1.0)
103
95
  regexp-examples (1.2.0)
104
- reportbuilder (1.4.2)
105
- clbustos-rtf (~> 0.4.0)
106
- prawn (~> 0.8.4)
107
- prawn-svg (~> 0.9.1)
108
- text-table (~> 1.2)
109
96
  restforce (2.2.0)
110
97
  faraday (~> 0.9.0)
111
98
  faraday_middleware (>= 0.8.8)
@@ -124,25 +111,25 @@ GEM
124
111
  diff-lcs (>= 1.2.0, < 2.0)
125
112
  rspec-support (~> 3.4.0)
126
113
  rspec-support (3.4.1)
127
- ruby-ole (1.2.12)
128
- spreadsheet (1.1.2)
129
- ruby-ole (>= 1.0)
130
- text-table (1.2.4)
131
114
  thread_safe (0.3.5)
132
115
  tzinfo (1.2.2)
133
116
  thread_safe (~> 0.1)
134
117
  xml-simple (1.1.5)
118
+ yard (0.9.0)
135
119
 
136
120
  PLATFORMS
137
121
  ruby
138
122
 
139
123
  DEPENDENCIES
140
124
  aws-sdk (~> 2.3)
141
- daru (= 0.1.2)!
125
+ daru (= 0.1.4.1)!
126
+ github-markup
142
127
  iruby (= 0.2.7)
128
+ redcarpet
143
129
  remi!
144
130
  restforce (~> 2.1)
145
131
  salesforce_bulk_api!
132
+ yard
146
133
 
147
134
  BUNDLED WITH
148
135
  1.11.2
data/README.md CHANGED
@@ -110,7 +110,7 @@ Remi::SourceToTargetMap.apply(df) do
110
110
  map source(:a, :b) .target(:aprime, :ab)
111
111
  .transform(->(row) {
112
112
  row[:aprime] = row[:a]
113
- row[:ab] = "#{row[:a]}#{row[:b]}" })
113
+ row[:ab] = "#{row[:a]}#{row[:b]}"
114
114
  })
115
115
  end
116
116
 
@@ -44,57 +44,65 @@ end
44
44
  ### Source file processing
45
45
 
46
46
  Given /^files with names matching the pattern \/(.*)\/$/ do |pattern|
47
- expect(@brt.source.data_subject.extractor.pattern).to eq Regexp.new(pattern)
47
+ expect(@brt.source.data_subject.extractors.map(&:pattern)).to include Regexp.new(pattern)
48
48
  end
49
49
 
50
50
  Given /^download groups defined by the pattern \/(.*)\/$/ do |pattern|
51
- expect(@brt.source.data_subject.extractor.group_by).to eq Regexp.new(pattern)
51
+ expect(@brt.source.data_subject.extractors.map(&:group_by)).to include Regexp.new(pattern)
52
52
  end
53
53
 
54
54
  Then /^the file with the latest date stamp will be downloaded for processing$/ do
55
- expect(@brt.source.data_subject.extractor.most_recent_by).to eq :create_time
55
+ expect(@brt.source.data_subject.extractors.map(&:most_recent_by)).to include :create_time
56
56
  end
57
57
 
58
58
  Then /^all files matching the pattern will be downloaded for processing$/ do
59
- expect(@brt.source.data_subject.extractor.most_recent_only).to eq false
59
+ expect(@brt.source.data_subject.extractors.map(&:most_recent_only)).not_to include eq true
60
60
  end
61
61
 
62
62
  Then /^the file that comes last in an alphanumeric sort by group will be downloaded for processing$/ do
63
- expect(@brt.source.data_subject.extractor.most_recent_by).to eq :name
63
+ expect(@brt.source.data_subject.extractors.map(&:most_recent_by)).to include :name
64
64
  end
65
65
 
66
66
  Then /^the file is uploaded to the remote path "([^"]+)"$/ do |remote_path|
67
- expect(@brt.target.get_attrib(:remote_path)).to eq Remi::Testing::BusinessRules::ParseFormula.parse(remote_path)
67
+ expected_path = Remi::Testing::BusinessRules::ParseFormula.parse(remote_path)
68
+ expect(@brt.target.data_subject.loaders.map(&:remote_path)).to include expected_path
68
69
  end
69
70
 
70
71
  ## CSV Options
71
72
 
72
73
  Given /^the (source|target) file is delimited with a (\w+)$/ do |st, delimiter|
73
- expect(@brt.send(st.to_sym).csv_options[:col_sep]).to eq Remi::Testing::BusinessRules.csv_opt_map[delimiter]
74
+ csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
75
+ expect(csv_subject.csv_options[:col_sep]).to eq Remi::Testing::BusinessRules.csv_opt_map[delimiter]
74
76
  end
75
77
 
76
78
  Given /^the (source|target) file is encoded using "([^"]+)" format$/ do |st, encoding|
77
- expect(@brt.send(st.to_sym).csv_options[:encoding].split(':').first).to eq encoding
79
+ csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
80
+ expect(csv_subject.csv_options[:encoding].split(':').first).to eq encoding
78
81
  end
79
82
 
80
83
  Given /^the (source|target) file uses a ([\w ]+) to quote embedded delimiters$/ do |st, quote_char|
81
- expect(@brt.send(st.to_sym).csv_options[:quote_char]).to eq Remi::Testing::BusinessRules.csv_opt_map[quote_char]
84
+ csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
85
+ expect(csv_subject.csv_options[:quote_char]).to eq Remi::Testing::BusinessRules.csv_opt_map[quote_char]
82
86
  end
83
87
 
84
88
  Given /^the (source|target) file uses a preceding ([\w ]+) to escape an embedded quoting character$/ do |st, escape_char|
85
- expect(@brt.send(st.to_sym).csv_options[:quote_char]).to eq Remi::Testing::BusinessRules.csv_opt_map[escape_char]
89
+ csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
90
+ expect(csv_subject.csv_options[:quote_char]).to eq Remi::Testing::BusinessRules.csv_opt_map[escape_char]
86
91
  end
87
92
 
88
93
  Given /^the (source|target) file uses ([\w ]+) line endings$/ do |st, line_endings|
89
- expect(@brt.send(st.to_sym).csv_options[:row_sep]).to eq Remi::Testing::BusinessRules.csv_opt_map[line_endings]
94
+ csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
95
+ expect(csv_subject.csv_options[:row_sep]).to eq Remi::Testing::BusinessRules.csv_opt_map[line_endings]
90
96
  end
91
97
 
92
98
  Given /^the (source|target) file uses "([^"]+)" as a record separator$/ do |st, line_endings|
93
- expect(@brt.send(st.to_sym).csv_options[:row_sep]).to eq line_endings.gsub(/\\n/, "\n").gsub(/\\r/, "\r")
99
+ csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
100
+ expect(csv_subject.csv_options[:row_sep]).to eq line_endings.gsub(/\\n/, "\n").gsub(/\\r/, "\r")
94
101
  end
95
102
 
96
103
  Given /^the (source|target) file (contains|does not contain) a header row$/ do |st, header|
97
- expect(@brt.send(st.to_sym).csv_options[:headers]).to eq (header == 'contains')
104
+ csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
105
+ expect(csv_subject.csv_options[:headers]).to eq (header == 'contains')
98
106
  end
99
107
 
100
108
  Given /^the (source|target) file contains at least the following headers in no particular order:$/ do |st, table|
@@ -301,6 +309,12 @@ Then /^the target '(.+)' should match the example '([[:alnum:]\-\s]+)'$/ do |tar
301
309
  target_hash = @brt.targets[target_name].column_hash
302
310
  example_hash = @brt.examples[example_name].column_hash
303
311
  common_keys = target_hash.keys & example_hash.keys
312
+ expect(common_keys).to match_array(example_hash.keys), <<-EOT
313
+ Fields in example not found in target
314
+ Example fields: #{example_hash.keys}
315
+ Target fields: #{target_hash.keys}
316
+ Missing fields: #{example_hash.keys - target_hash.keys}
317
+ EOT
304
318
 
305
319
  target_to_compare = target_hash.select { |k,v| common_keys.include? k }
306
320
  target_to_compare.each do |k, v|
@@ -317,6 +331,12 @@ Then /^the target should match the example '([[:alnum:]\-\s]+)'$/ do |example_na
317
331
  step "the target '#{target_name}' should match the example '#{example_name}'"
318
332
  end
319
333
 
334
+ Then /^the target '(.+)' should match the example:/ do |target_name, example_table|
335
+ example_name = SecureRandom.uuid
336
+ @brt.add_example example_name, example_table
337
+ step "the target '#{target_name}' should match the example '#{example_name}'"
338
+ end
339
+
320
340
  Then /^the target should match the example:/ do |example_table|
321
341
  example_name = SecureRandom.uuid
322
342
  @brt.add_example example_name, example_table
@@ -0,0 +1,24 @@
1
+ Feature: This is a job that demonstrates how to use Remi subjobs
2
+
3
+ Background:
4
+ Given the job is 'Sub Job Example'
5
+ And the job target 'Just Beers'
6
+ And the job target 'Zombified Beers'
7
+
8
+ Scenario: Sub jobs can be used to extract data
9
+
10
+ Given the target 'Just Beers'
11
+
12
+ Then the target should match the example:
13
+ | Brewer | Style |
14
+ | Baerlic | IPA |
15
+ | Ex Novo | Red |
16
+
17
+ Scenario: Sub jobs can be used to transform source data
18
+
19
+ Given the target 'Zombified Beers'
20
+
21
+ Then the target should match the example:
22
+ | Brewer | Style |
23
+ | Zombie Baerlic | Zombie IPA |
24
+ | Zombie Ex Novo | Zombie Red |
@@ -0,0 +1,35 @@
1
+ Feature: This is a job that demonstrates how to use Remi sub transforms
2
+
3
+ Background:
4
+ Given the job is 'Sub Transform Example'
5
+ And the job source 'My Source'
6
+ And the job target 'My Target'
7
+
8
+ Scenario: It uses the subtransform to prefix a field with the default value
9
+
10
+ Given the source 'My Source'
11
+ And the target 'My Target'
12
+
13
+ And the following example for 'My Source':
14
+ | id |
15
+ | 1 |
16
+
17
+ Then the target should match the example:
18
+ | id | default_id |
19
+ | 1 | DEFAULT1 |
20
+
21
+
22
+ Scenario: It uses parameters to customize subtansforms
23
+
24
+ Given the source 'My Source'
25
+ And the target 'My Target'
26
+
27
+ And the job parameter 'job_prefix' is "UC"
28
+
29
+ And the following example for 'My Source':
30
+ | id |
31
+ | 1 |
32
+
33
+ Then the target should match the example:
34
+ | id | default_id |
35
+ | 1 | UC1 |
@@ -0,0 +1,49 @@
1
+ Feature: This is a job that demonstrates how to use Remi sub transforms
2
+
3
+ Background:
4
+ Given the job is 'Sub Transform Many To Many'
5
+ And the job source 'Beer Fact'
6
+ And the job source 'Beer Dim'
7
+ And the job target 'Beer Count'
8
+ And the job target 'Style Count'
9
+
10
+
11
+ Scenario: It runs the subtransform as expected
12
+
13
+ Given the source 'Beer Fact'
14
+ And the source 'Beer Dim'
15
+ And the target 'Beer Count'
16
+ And the target 'Style Count'
17
+
18
+ And the following example for 'Beer Fact':
19
+ | fact_sk | beer_sk |
20
+ | 1 | 1 |
21
+ | 2 | 1 |
22
+ | 3 | 1 |
23
+ | 4 | 2 |
24
+ | 5 | 2 |
25
+ | 6 | 2 |
26
+ | 7 | 3 |
27
+ | 8 | 3 |
28
+ | 9 | 3 |
29
+ | 10 | 4 |
30
+
31
+ And the following example for 'Beer Dim':
32
+ | beer_sk | name | style |
33
+ | 1 | Invincible | IPA |
34
+ | 2 | Pipewrench | IPA |
35
+ | 3 | Altera | Red |
36
+ | 4 | Urban Farmhouse | Farmhouse |
37
+
38
+ Then the target 'Beer Count' should match the example:
39
+ | name | count |
40
+ | Altera | 3 |
41
+ | Invincible | 3 |
42
+ | Pipewrench | 3 |
43
+ | Urban Farmhouse | 1 |
44
+
45
+ And the target 'Style Count' should match the example:
46
+ | style | count |
47
+ | Farmhouse | 1 |
48
+ | IPA | 6 |
49
+ | Red | 3 |
@@ -1 +1 @@
1
- Dir["#{File.dirname(__FILE__)}/../../jobs/**/*_job.rb"].each { |f| require(f) }
1
+ # Put stuff here that is specific to your testing environment
@@ -4,22 +4,25 @@
4
4
  require 'bundler/setup'
5
5
  require 'remi'
6
6
 
7
- module AllJobsShared
8
- include Remi::Job
7
+ class AllJobsShared < Remi::Job
9
8
 
10
- define_param :sftp, {
11
- host: 'example.com',
12
- username: 'user',
13
- password: '1234567890'
14
- }
9
+ param :sftp do
10
+ {
11
+ host: 'example.com',
12
+ username: 'user',
13
+ password: '1234567890'
14
+ }
15
+ end
15
16
 
16
- define_param :salesforce_credentials, {
17
- host: 'login.salesforce.com',
18
- username: 'user@example.com',
19
- password: 'password',
20
- security_token: '4342jn3j4n32n4',
21
- client_id: 'dkfjsdkfjoasdjdf',
22
- client_secret: '28357245723475',
23
- instance_url: 'https://na1.salesforce.com'
24
- }
17
+ param :salesforce_credentials do
18
+ {
19
+ host: 'login.salesforce.com',
20
+ username: 'user@example.com',
21
+ password: 'password',
22
+ security_token: '4342jn3j4n32n4',
23
+ client_id: 'dkfjsdkfjoasdjdf',
24
+ client_secret: '28357245723475',
25
+ instance_url: 'https://na1.salesforce.com'
26
+ }
27
+ end
25
28
  end
@@ -1,16 +1,18 @@
1
1
  require_relative 'all_jobs_shared'
2
2
 
3
- class CopySourceJob
4
- include AllJobsShared
3
+ class CopySourceJob < Remi::Job
4
+ source :source_data do
5
+ fields(
6
+ {
7
+ :some_field => {},
8
+ :some_date => { type: :date, format: '%Y-%m-%d' }
9
+ }
10
+ )
11
+ end
5
12
 
6
- define_source :source_data, Remi::DataSource::DataFrame,
7
- fields: {
8
- :some_field => {},
9
- :some_date => { type: :date, format: '%Y-%m-%d' }
10
- }
11
- define_source :target_data, Remi::DataSource::DataFrame
13
+ target :target_data
12
14
 
13
- define_transform :main, sources: :source_data, targets: :target_data do
15
+ transform :main do
14
16
  target_data.df = source_data.df.dup
15
17
  end
16
18
  end
@@ -1,15 +1,16 @@
1
1
  require_relative 'all_jobs_shared'
2
2
 
3
- class CsvFileTargetJob
4
- include AllJobsShared
5
-
6
- define_target :some_csv_file, Remi::DataTarget::CsvFile,
7
- path: "#{Remi::Settings.work_dir}/some_file.csv",
8
- csv_options: {
9
- col_sep: '|'
10
- }
3
+ class CsvFileTargetJob < Remi::Job
4
+ target :some_csv_file do
5
+ encoder Remi::Encoder::CsvFile.new(
6
+ path: "#{Remi::Settings.work_dir}/some_file.csv",
7
+ csv_options: {
8
+ col_sep: '|'
9
+ }
10
+ )
11
+ end
11
12
 
12
- define_transform :main do
13
+ transform :main do
13
14
  some_csv_file.df = Daru::DataFrame.new({
14
15
  col3: Faker::Hipster.words(10),
15
16
  col1: Faker::Hipster.words(10),