data_forge 0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +19 -0
- data/.rspec +2 -0
- data/.travis.yml +5 -0
- data/Gemfile +2 -0
- data/LICENSE +20 -0
- data/README.md +49 -0
- data/Rakefile +16 -0
- data/bin/forge +4 -0
- data/config/cucumber.yml +8 -0
- data/data_forge.gemspec +26 -0
- data/features/accessing_command_line_parameters.feature +52 -0
- data/features/deduplication.feature +49 -0
- data/features/file/file_format_options.feature +146 -0
- data/features/file/has_header_row.feature +62 -0
- data/features/step_definitions/file_steps.rb +8 -0
- data/features/support/env.rb +8 -0
- data/features/transform/output_command.feature +123 -0
- data/features/transform/outputting_to_multiple_files.feature +57 -0
- data/features/transform/overwrite_original_file.feature +37 -0
- data/features/transform/record_transformation.feature +47 -0
- data/lib/data_forge/cli/main.rb +21 -0
- data/lib/data_forge/cli/options.rb +62 -0
- data/lib/data_forge/cli.rb +24 -0
- data/lib/data_forge/dsl/attributes.rb +15 -0
- data/lib/data_forge/dsl/commands.rb +23 -0
- data/lib/data_forge/dsl/helpers.rb +22 -0
- data/lib/data_forge/dsl.rb +9 -0
- data/lib/data_forge/file/csv/csv_record_file_definition.rb +46 -0
- data/lib/data_forge/file/csv/csv_record_file_reader.rb +42 -0
- data/lib/data_forge/file/csv/csv_record_file_writer.rb +62 -0
- data/lib/data_forge/file/csv.rb +13 -0
- data/lib/data_forge/file/record_file_definition.rb +17 -0
- data/lib/data_forge/file/record_file_reader.rb +22 -0
- data/lib/data_forge/file/record_file_writer.rb +32 -0
- data/lib/data_forge/file.rb +36 -0
- data/lib/data_forge/transformation/deduplication.rb +38 -0
- data/lib/data_forge/transformation/ruby_transformation.rb +33 -0
- data/lib/data_forge/transformation/ruby_transformation_context.rb +27 -0
- data/lib/data_forge/transformation/transformation_base.rb +29 -0
- data/lib/data_forge/transformation.rb +10 -0
- data/lib/data_forge/version.rb +3 -0
- data/lib/data_forge.rb +13 -0
- data/spec/data_forge/cli/main_spec.rb +45 -0
- data/spec/data_forge/cli/options_spec.rb +64 -0
- data/spec/data_forge/cli_spec.rb +54 -0
- data/spec/data_forge/dsl/commands_spec.rb +42 -0
- data/spec/data_forge/dsl/helpers_spec.rb +24 -0
- data/spec/data_forge/file/csv/csv_record_file_definition_spec.rb +97 -0
- data/spec/data_forge/file/csv/csv_record_file_reader_spec.rb +78 -0
- data/spec/data_forge/file/csv/csv_record_file_writer_spec.rb +100 -0
- data/spec/data_forge/file/record_file_definition_spec.rb +17 -0
- data/spec/data_forge/file/record_file_reader_spec.rb +15 -0
- data/spec/data_forge/file/record_file_writer_spec.rb +15 -0
- data/spec/data_forge/file_spec.rb +49 -0
- data/spec/data_forge/transformation/deduplication_spec.rb +77 -0
- data/spec/data_forge/transformation/ruby_transformation_context_spec.rb +49 -0
- data/spec/data_forge/transformation/ruby_transformation_spec.rb +71 -0
- data/spec/data_forge_spec.rb +9 -0
- data/spec/spec_helper.rb +17 -0
- data/spec/support/helpers/record_reader_helper.rb +17 -0
- data/spec/support/helpers/record_writer_helper.rb +16 -0
- metadata +218 -0
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2013 Zoltan Ormandi
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
6
|
+
this software and associated documentation files (the "Software"), to deal in
|
7
|
+
the Software without restriction, including without limitation the rights to
|
8
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
9
|
+
the Software, and to permit persons to whom the Software is furnished to do so,
|
10
|
+
subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
17
|
+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
18
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
19
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
20
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,49 @@
|
|
1
|
+
[![Build Status](https://travis-ci.org/zormandi/data_forge.svg)](https://travis-ci.org/zormandi/data_forge)
|
2
|
+
|
3
|
+
# DataForge
|
4
|
+
|
5
|
+
DataForge is a data manipulation tool for transferring (and transforming) data between flat files and databases.
|
6
|
+
|
7
|
+
## Requirements
|
8
|
+
|
9
|
+
* Ruby >= 1.9
|
10
|
+
|
11
|
+
## Installation
|
12
|
+
|
13
|
+
Add this line to your application's Gemfile:
|
14
|
+
|
15
|
+
``` ruby
|
16
|
+
gem 'data_forge'
|
17
|
+
```
|
18
|
+
|
19
|
+
And then execute:
|
20
|
+
|
21
|
+
``` sh
|
22
|
+
$ bundle
|
23
|
+
```
|
24
|
+
|
25
|
+
Or install it yourself as:
|
26
|
+
|
27
|
+
``` sh
|
28
|
+
$ gem install data_forge
|
29
|
+
```
|
30
|
+
|
31
|
+
## Usage
|
32
|
+
|
33
|
+
``` sh
|
34
|
+
$ forge [options] <command script>
|
35
|
+
```
|
36
|
+
|
37
|
+
For information on how to use Data Forge please see the `features` directory, which has code examples on every working
|
38
|
+
feature of the gem.
|
39
|
+
|
40
|
+
For the complete documentation (coming with the 0.2 version) and the detailed development roadmap, please see the
|
41
|
+
[Wiki](https://github.com/zormandi/data_forge/wiki).
|
42
|
+
|
43
|
+
## Contributing
|
44
|
+
|
45
|
+
1. Fork it
|
46
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
47
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
48
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
49
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
require "rspec/core/rake_task"
|
3
|
+
require 'cucumber/rake/task'
|
4
|
+
|
5
|
+
desc "Run RSpec code examples (options: RSPEC_SEED=seed)"
|
6
|
+
RSpec::Core::RakeTask.new :spec do |task|
|
7
|
+
task.verbose = false
|
8
|
+
task.rspec_opts = "--format progress --order random"
|
9
|
+
task.rspec_opts << " --seed #{ENV['RSPEC_SEED']}" if ENV['RSPEC_SEED']
|
10
|
+
end
|
11
|
+
|
12
|
+
Cucumber::Rake::Task.new(:features, "Run Cucumber features") do |task|
|
13
|
+
task.cucumber_opts = %w[--profile build]
|
14
|
+
end
|
15
|
+
|
16
|
+
task :default => [:spec, :features]
|
data/bin/forge
ADDED
data/config/cucumber.yml
ADDED
data/data_forge.gemspec
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'data_forge/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "data_forge"
|
8
|
+
spec.version = DataForge::VERSION
|
9
|
+
spec.authors = ["Zoltan Ormandi"]
|
10
|
+
spec.email = ["zoltan.ormandi@gmail.com"]
|
11
|
+
spec.description = %q{DataForge is a data manipulation tool for transferring (and transforming) data between flat files and databases.}
|
12
|
+
spec.summary = %q{Pure Ruby ETL and data manipulation tool.}
|
13
|
+
spec.homepage = "https://github.com/zormandi/data_forge"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files`.split($/)
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
22
|
+
spec.add_development_dependency "rake"
|
23
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
24
|
+
spec.add_development_dependency "cucumber"
|
25
|
+
spec.add_development_dependency "aruba"
|
26
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
Feature: Accessing command line parameters
|
2
|
+
|
3
|
+
The following command line parameters are available in the command script during execution:
|
4
|
+
- COMMAND_SCRIPT: The name of the command script currently executing.
|
5
|
+
- PARAMS: The user-defined parameters passed to the script as a hash.
|
6
|
+
|
7
|
+
|
8
|
+
Scenario: Accessing the command script
|
9
|
+
Given a file named "config.rb" with:
|
10
|
+
"""
|
11
|
+
File.write 'script_name.txt', COMMAND_SCRIPT
|
12
|
+
"""
|
13
|
+
And a file named "command_script.rb" with:
|
14
|
+
"""
|
15
|
+
require_relative 'config'
|
16
|
+
"""
|
17
|
+
When I run `forge command_script.rb`
|
18
|
+
Then the exit status should be 0
|
19
|
+
And a file named "script_name.txt" should exist
|
20
|
+
And the file "script_name.txt" should contain "command_script.rb"
|
21
|
+
|
22
|
+
|
23
|
+
Scenario: Passing user-defined parameters to a file definition and a transform block
|
24
|
+
Given a file named "command_script.rb" with:
|
25
|
+
"""
|
26
|
+
file :items do
|
27
|
+
file_name PARAMS[:data_file]
|
28
|
+
field :id
|
29
|
+
end
|
30
|
+
|
31
|
+
transform :items do |record|
|
32
|
+
record[:id] = PARAMS[:id]
|
33
|
+
output record
|
34
|
+
end
|
35
|
+
"""
|
36
|
+
And a file named "ids.csv" with:
|
37
|
+
"""
|
38
|
+
id
|
39
|
+
1
|
40
|
+
2
|
41
|
+
3
|
42
|
+
"""
|
43
|
+
When I run `forge -Udata_file=ids.csv -Uid=5 command_script.rb`
|
44
|
+
Then the exit status should be 0
|
45
|
+
And the file "ids.csv" should contain exactly:
|
46
|
+
"""
|
47
|
+
id
|
48
|
+
5
|
49
|
+
5
|
50
|
+
5
|
51
|
+
|
52
|
+
"""
|
@@ -0,0 +1,49 @@
|
|
1
|
+
Feature: Deduplicating data in a file
|
2
|
+
|
3
|
+
The `deduplicate` keyword can be used to remove duplicate records from a file. Only the first occurence of each
|
4
|
+
duplicate record is kept.
|
5
|
+
|
6
|
+
Parameters:
|
7
|
+
- The source file to be deduplicated. This parameter is mandatory.
|
8
|
+
- into: The target file name. If no target file is specified then the source file is overwritten with the result
|
9
|
+
of the deduplication.
|
10
|
+
- using: The fields to be used to determine whether or not a record is a duplicate. If not specified then all
|
11
|
+
fields of the source file are used.
|
12
|
+
|
13
|
+
|
14
|
+
Scenario: Single file transformation
|
15
|
+
Given a file named "command_script.rb" with:
|
16
|
+
"""
|
17
|
+
file :items do
|
18
|
+
field :rownum
|
19
|
+
field :item_id
|
20
|
+
field :item_name
|
21
|
+
end
|
22
|
+
|
23
|
+
file :unique_items do
|
24
|
+
field :item_id
|
25
|
+
field :item_name
|
26
|
+
end
|
27
|
+
|
28
|
+
deduplicate :items, into: :unique_items, using: :item_id
|
29
|
+
"""
|
30
|
+
And a file named "items.csv" with:
|
31
|
+
"""
|
32
|
+
rownum,item_id,item_name
|
33
|
+
1,Item1,Item name 1
|
34
|
+
2,Item1,Item name 1
|
35
|
+
3,Item2,Item name 2
|
36
|
+
4,Item2,Item name 2
|
37
|
+
5,Item3,Item name 3
|
38
|
+
"""
|
39
|
+
When I run `forge command_script.rb`
|
40
|
+
Then the exit status should be 0
|
41
|
+
And a file named "unique_items.csv" should exist
|
42
|
+
And the file "unique_items.csv" should contain exactly:
|
43
|
+
"""
|
44
|
+
item_id,item_name
|
45
|
+
Item1,Item name 1
|
46
|
+
Item2,Item name 2
|
47
|
+
Item3,Item name 3
|
48
|
+
|
49
|
+
"""
|
@@ -0,0 +1,146 @@
|
|
1
|
+
Feature: Formatting options for CSV files
|
2
|
+
|
3
|
+
The `file` block offers options for setting the CSV file's delimiter character, quote character and encoding.
|
4
|
+
All native Ruby encodings are supported.
|
5
|
+
|
6
|
+
The default values are:
|
7
|
+
- delimiter: ,
|
8
|
+
- quote character: "
|
9
|
+
- encoding: UTF-8
|
10
|
+
|
11
|
+
|
12
|
+
Scenario: Using the default delimiter and quote character
|
13
|
+
Given a file named "command_script.rb" with:
|
14
|
+
"""
|
15
|
+
file :items do
|
16
|
+
field :id
|
17
|
+
field :name
|
18
|
+
field :category
|
19
|
+
end
|
20
|
+
|
21
|
+
file :items_copy do
|
22
|
+
field :id
|
23
|
+
field :name
|
24
|
+
field :category
|
25
|
+
end
|
26
|
+
|
27
|
+
transform :items, into: :items_copy do |record|
|
28
|
+
output record
|
29
|
+
end
|
30
|
+
"""
|
31
|
+
And a file named "items.csv" with:
|
32
|
+
"""
|
33
|
+
id,name,category
|
34
|
+
1,"27"" screen",Screens
|
35
|
+
"""
|
36
|
+
When I run `forge command_script.rb`
|
37
|
+
Then the exit status should be 0
|
38
|
+
And a file named "items_copy.csv" should exist
|
39
|
+
And the file "items_copy.csv" should contain exactly:
|
40
|
+
"""
|
41
|
+
id,name,category
|
42
|
+
1,"27"" screen",Screens
|
43
|
+
|
44
|
+
"""
|
45
|
+
|
46
|
+
|
47
|
+
Scenario: Using custom delimiter and quote character
|
48
|
+
Given a file named "command_script.rb" with:
|
49
|
+
"""
|
50
|
+
file :items do
|
51
|
+
delimiter ";"
|
52
|
+
quote "'"
|
53
|
+
|
54
|
+
field :id
|
55
|
+
field :name
|
56
|
+
field :category
|
57
|
+
end
|
58
|
+
|
59
|
+
file :items_copy do
|
60
|
+
delimiter "|"
|
61
|
+
quote "`"
|
62
|
+
|
63
|
+
field :id
|
64
|
+
field :name
|
65
|
+
field :category
|
66
|
+
end
|
67
|
+
|
68
|
+
transform :items, into: :items_copy do |record|
|
69
|
+
output record
|
70
|
+
end
|
71
|
+
"""
|
72
|
+
And a file named "items.csv" with:
|
73
|
+
"""
|
74
|
+
id;name;category
|
75
|
+
1;'27'' screen';Screens
|
76
|
+
2;24` screen;Screens
|
77
|
+
"""
|
78
|
+
When I run `forge command_script.rb`
|
79
|
+
Then the exit status should be 0
|
80
|
+
And a file named "items_copy.csv" should exist
|
81
|
+
And the file "items_copy.csv" should contain exactly:
|
82
|
+
"""
|
83
|
+
id|name|category
|
84
|
+
1|27' screen|Screens
|
85
|
+
2|`24`` screen`|Screens
|
86
|
+
|
87
|
+
"""
|
88
|
+
|
89
|
+
|
90
|
+
Scenario: Using custom encoding
|
91
|
+
Given a file named "command_script.rb" with:
|
92
|
+
"""
|
93
|
+
file :items do
|
94
|
+
encoding "UTF-8"
|
95
|
+
|
96
|
+
field :id
|
97
|
+
field :name
|
98
|
+
field :category
|
99
|
+
end
|
100
|
+
|
101
|
+
file :items_latin2 do
|
102
|
+
encoding "ISO-8859-2"
|
103
|
+
|
104
|
+
field :id
|
105
|
+
field :name
|
106
|
+
field :category
|
107
|
+
end
|
108
|
+
|
109
|
+
file :items_copy do
|
110
|
+
encoding "UTF-8"
|
111
|
+
|
112
|
+
field :id
|
113
|
+
field :name
|
114
|
+
field :category
|
115
|
+
end
|
116
|
+
|
117
|
+
transform :items, into: :items_latin2 do |record|
|
118
|
+
output record
|
119
|
+
end
|
120
|
+
|
121
|
+
transform :items_latin2, into: :items_copy do |record|
|
122
|
+
output record
|
123
|
+
end
|
124
|
+
"""
|
125
|
+
And a file named "items.csv" with:
|
126
|
+
"""
|
127
|
+
id,name,category
|
128
|
+
1,Képernyő,Screens
|
129
|
+
"""
|
130
|
+
When I run `forge command_script.rb`
|
131
|
+
Then the exit status should be 0
|
132
|
+
And the following files should exist:
|
133
|
+
| items_latin2.csv |
|
134
|
+
| items_copy.csv |
|
135
|
+
And the file "items_latin2.csv" should contain in "ISO-8859-2" encoding exactly:
|
136
|
+
"""
|
137
|
+
id,name,category
|
138
|
+
1,Képernyő,Screens
|
139
|
+
|
140
|
+
"""
|
141
|
+
And the file "items_copy.csv" should contain in "UTF-8" encoding exactly:
|
142
|
+
"""
|
143
|
+
id,name,category
|
144
|
+
1,Képernyő,Screens
|
145
|
+
|
146
|
+
"""
|
@@ -0,0 +1,62 @@
|
|
1
|
+
Feature: File `has_header` option
|
2
|
+
|
3
|
+
The `has_header_row` option of a `file` block specifies whether or not the corresponding CSV file has a header row.
|
4
|
+
If a file is specified to have a header row then the first row of the file is skipped during transformation.
|
5
|
+
If not, then all rows of the file are processed. When writing into files with a header row, the fields specified
|
6
|
+
for that file are written as the header of the CSV file. If an output file is specified to have no header then
|
7
|
+
only data rows are written into it.
|
8
|
+
|
9
|
+
|
10
|
+
Scenario: Using the has_header option
|
11
|
+
Given a file named "command_script.rb" with:
|
12
|
+
"""
|
13
|
+
file :items do
|
14
|
+
has_header_row true
|
15
|
+
|
16
|
+
field :id
|
17
|
+
field :name
|
18
|
+
end
|
19
|
+
|
20
|
+
file :items_without_header do
|
21
|
+
has_header_row false
|
22
|
+
|
23
|
+
field :id
|
24
|
+
field :name
|
25
|
+
end
|
26
|
+
|
27
|
+
file :items_copy do
|
28
|
+
has_header_row true
|
29
|
+
|
30
|
+
field :id
|
31
|
+
field :name
|
32
|
+
end
|
33
|
+
|
34
|
+
transform :items, into: :items_without_header do |record|
|
35
|
+
output record
|
36
|
+
end
|
37
|
+
|
38
|
+
transform :items_without_header, into: :items_copy do |record|
|
39
|
+
output record
|
40
|
+
end
|
41
|
+
"""
|
42
|
+
And a file named "items.csv" with:
|
43
|
+
"""
|
44
|
+
id,name
|
45
|
+
1,data
|
46
|
+
"""
|
47
|
+
When I run `forge command_script.rb`
|
48
|
+
Then the exit status should be 0
|
49
|
+
And the following files should exist:
|
50
|
+
| items_without_header.csv |
|
51
|
+
| items_copy.csv |
|
52
|
+
And the file "items_without_header.csv" should contain exactly:
|
53
|
+
"""
|
54
|
+
1,data
|
55
|
+
|
56
|
+
"""
|
57
|
+
And the file "items_copy.csv" should contain exactly:
|
58
|
+
"""
|
59
|
+
id,name
|
60
|
+
1,data
|
61
|
+
|
62
|
+
"""
|
@@ -0,0 +1,8 @@
|
|
1
|
+
Then /^the file "([^"]*)" should be empty$/ do |file_name|
|
2
|
+
step %Q(the file "#{file_name}" should contain exactly:), ""
|
3
|
+
end
|
4
|
+
|
5
|
+
|
6
|
+
Then /^the file "([^"]*)" should contain in "([^"]*)" encoding exactly:$/ do |file_name, encoding, content|
|
7
|
+
prep_for_fs_check { expect(IO.read(file_name, encoding: encoding).encode("UTF-8", encoding)).to eq content }
|
8
|
+
end
|
@@ -0,0 +1,123 @@
|
|
1
|
+
Feature: Using the `output` command
|
2
|
+
|
3
|
+
The `output` command in a transformation block writes the specified data (HasH) into the output file.
|
4
|
+
Omitting this command conditionally for a row will skip the record and omitting it unconditionally will
|
5
|
+
simply create an empty file without outputting any data into it.
|
6
|
+
|
7
|
+
|
8
|
+
Background:
|
9
|
+
Given a file named "items.csv" with:
|
10
|
+
"""
|
11
|
+
id
|
12
|
+
1
|
13
|
+
2
|
14
|
+
3
|
15
|
+
"""
|
16
|
+
|
17
|
+
|
18
|
+
Scenario: Outputting arbitrary data as Hash
|
19
|
+
Given a file named "command_script.rb" with:
|
20
|
+
"""
|
21
|
+
file :items do
|
22
|
+
field :id
|
23
|
+
end
|
24
|
+
|
25
|
+
file :books do
|
26
|
+
field :author
|
27
|
+
field :title
|
28
|
+
end
|
29
|
+
|
30
|
+
transform :items, into: :books do |record|
|
31
|
+
output title: "Title #{record[:id]}",
|
32
|
+
author: "Author #{record[:id]}"
|
33
|
+
end
|
34
|
+
"""
|
35
|
+
When I run `forge command_script.rb`
|
36
|
+
Then the exit status should be 0
|
37
|
+
And a file named "books.csv" should exist
|
38
|
+
And the file "books.csv" should contain exactly:
|
39
|
+
"""
|
40
|
+
author,title
|
41
|
+
Author 1,Title 1
|
42
|
+
Author 2,Title 2
|
43
|
+
Author 3,Title 3
|
44
|
+
|
45
|
+
"""
|
46
|
+
|
47
|
+
|
48
|
+
Scenario: No output in transform block creates empty file
|
49
|
+
Given a file named "command_script.rb" with:
|
50
|
+
"""
|
51
|
+
file :items do
|
52
|
+
field :id
|
53
|
+
end
|
54
|
+
|
55
|
+
file :items_empty do
|
56
|
+
field :id
|
57
|
+
end
|
58
|
+
|
59
|
+
transform :items, into: :items_empty do end
|
60
|
+
"""
|
61
|
+
When I run `forge command_script.rb`
|
62
|
+
Then the exit status should be 0
|
63
|
+
And a file named "items_empty.csv" should exist
|
64
|
+
And the file "items_empty.csv" should be empty
|
65
|
+
|
66
|
+
|
67
|
+
Scenario: Omitting output conditionally skips current record
|
68
|
+
Given a file named "command_script.rb" with:
|
69
|
+
"""
|
70
|
+
file :items do
|
71
|
+
field :id
|
72
|
+
end
|
73
|
+
|
74
|
+
file :items_missing do
|
75
|
+
field :id
|
76
|
+
end
|
77
|
+
|
78
|
+
transform :items, into: :items_missing do |record|
|
79
|
+
output record unless "2" == record[:id]
|
80
|
+
end
|
81
|
+
"""
|
82
|
+
When I run `forge command_script.rb`
|
83
|
+
Then the exit status should be 0
|
84
|
+
And a file named "items_missing.csv" should exist
|
85
|
+
And the file "items_missing.csv" should contain exactly:
|
86
|
+
"""
|
87
|
+
id
|
88
|
+
1
|
89
|
+
3
|
90
|
+
|
91
|
+
"""
|
92
|
+
|
93
|
+
|
94
|
+
Scenario: Multiple output commands multiply lines
|
95
|
+
Given a file named "command_script.rb" with:
|
96
|
+
"""
|
97
|
+
file :items do
|
98
|
+
field :id
|
99
|
+
end
|
100
|
+
|
101
|
+
file :items_doubled do
|
102
|
+
field :id
|
103
|
+
end
|
104
|
+
|
105
|
+
transform :items, into: :items_doubled do |record|
|
106
|
+
output record
|
107
|
+
output record
|
108
|
+
end
|
109
|
+
"""
|
110
|
+
When I run `forge command_script.rb`
|
111
|
+
Then the exit status should be 0
|
112
|
+
And a file named "items_doubled.csv" should exist
|
113
|
+
And the file "items_doubled.csv" should contain exactly:
|
114
|
+
"""
|
115
|
+
id
|
116
|
+
1
|
117
|
+
1
|
118
|
+
2
|
119
|
+
2
|
120
|
+
3
|
121
|
+
3
|
122
|
+
|
123
|
+
"""
|
@@ -0,0 +1,57 @@
|
|
1
|
+
Feature: Splitting a file into multiple files
|
2
|
+
|
3
|
+
The `transform` block can take multiple output file parameters, in which case the `output` command can be used
|
4
|
+
to specify which file to write to.
|
5
|
+
|
6
|
+
|
7
|
+
Scenario:
|
8
|
+
Given a file named "command_script.rb" with:
|
9
|
+
"""
|
10
|
+
file :items do
|
11
|
+
field :name
|
12
|
+
end
|
13
|
+
|
14
|
+
file :items_not_a do
|
15
|
+
field :name
|
16
|
+
end
|
17
|
+
|
18
|
+
file :items_not_b do
|
19
|
+
field :name
|
20
|
+
end
|
21
|
+
|
22
|
+
transform :items, into: [:items_not_a, :items_not_b] do |record|
|
23
|
+
if record[:name].start_with? "a"
|
24
|
+
output record, to: :items_not_b
|
25
|
+
elsif record[:name].start_with? "b"
|
26
|
+
output record, to: :items_not_a
|
27
|
+
else
|
28
|
+
output record, to: [:items_not_a, :items_not_b]
|
29
|
+
end
|
30
|
+
end
|
31
|
+
"""
|
32
|
+
And a file named "items.csv" with:
|
33
|
+
"""
|
34
|
+
name
|
35
|
+
a
|
36
|
+
b
|
37
|
+
c
|
38
|
+
"""
|
39
|
+
When I run `forge command_script.rb`
|
40
|
+
Then the exit status should be 0
|
41
|
+
And the following files should exist:
|
42
|
+
| items_not_a.csv |
|
43
|
+
| items_not_b.csv |
|
44
|
+
And the file "items_not_a.csv" should contain exactly:
|
45
|
+
"""
|
46
|
+
name
|
47
|
+
b
|
48
|
+
c
|
49
|
+
|
50
|
+
"""
|
51
|
+
And the file "items_not_b.csv" should contain exactly:
|
52
|
+
"""
|
53
|
+
name
|
54
|
+
a
|
55
|
+
c
|
56
|
+
|
57
|
+
"""
|