data_forge 0.1

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+.ruby-version
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.idea

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - "1.9.3"
+  - "2.0.0"
+  - "2.1.2"

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Zoltan Ormandi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,49 @@
+[![Build Status](https://travis-ci.org/zormandi/data_forge.svg)](https://travis-ci.org/zormandi/data_forge)
+
+# DataForge
+
+DataForge is a data manipulation tool for transferring (and transforming) data between flat files and databases.
+
+## Requirements
+
+* Ruby >= 1.9
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+``` ruby
+gem 'data_forge'
+```
+
+And then execute:
+
+``` sh
+$ bundle
+```
+
+Or install it yourself as:
+
+``` sh
+$ gem install data_forge
+```
+
+## Usage
+
+``` sh
+$ forge [options] <command script>
+```
+
+For information on how to use Data Forge please see the `features` directory, which has code examples on every working
+feature of the gem.
+
+For the complete documentation (coming with the 0.2 version) and the detailed development roadmap, please see the 
+[Wiki](https://github.com/zormandi/data_forge/wiki).
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,16 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+require 'cucumber/rake/task'
+
+desc "Run RSpec code examples (options: RSPEC_SEED=seed)"
+RSpec::Core::RakeTask.new :spec do |task|
+  task.verbose = false
+  task.rspec_opts = "--format progress --order random"
+  task.rspec_opts << " --seed #{ENV['RSPEC_SEED']}" if ENV['RSPEC_SEED']
+end
+
+Cucumber::Rake::Task.new(:features, "Run Cucumber features") do |task|
+  task.cucumber_opts = %w[--profile build]
+end
+
+task :default => [:spec, :features]

data/bin/forge

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'data_forge'
+
+DataForge::CLI::Main.new(ARGV.dup).execute!

data/config/cucumber.yml

@@ -0,0 +1,8 @@
+default:
+  --backtrace
+  --tags ~@wip
+
+build:
+  --backtrace
+  --format progress
+  --tags ~@wip

data/data_forge.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'data_forge/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "data_forge"
+  spec.version       = DataForge::VERSION
+  spec.authors       = ["Zoltan Ormandi"]
+  spec.email         = ["zoltan.ormandi@gmail.com"]
+  spec.description   = %q{DataForge is a data manipulation tool for transferring (and transforming) data between flat files and databases.}
+  spec.summary       = %q{Pure Ruby ETL and data manipulation tool.}
+  spec.homepage      = "https://github.com/zormandi/data_forge"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "cucumber"
+  spec.add_development_dependency "aruba"
+end

data/features/accessing_command_line_parameters.feature

@@ -0,0 +1,52 @@
+Feature: Accessing command line parameters
+
+  The following command line parameters are available in the command script during execution:
+  - COMMAND_SCRIPT: The name of the command script currently executing.
+  - PARAMS: The user-defined parameters passed to the script as a hash.
+
+
+  Scenario: Accessing the command script
+    Given a file named "config.rb" with:
+    """
+    File.write 'script_name.txt', COMMAND_SCRIPT
+    """
+    And a file named "command_script.rb" with:
+    """
+    require_relative 'config'
+    """
+    When I run `forge command_script.rb`
+    Then the exit status should be 0
+    And a file named "script_name.txt" should exist
+    And the file "script_name.txt" should contain "command_script.rb"
+
+
+  Scenario: Passing user-defined parameters to a file definition and a transform block
+    Given a file named "command_script.rb" with:
+    """
+    file :items do
+      file_name PARAMS[:data_file]
+      field :id
+    end
+
+    transform :items do |record|
+      record[:id] = PARAMS[:id]
+      output record
+    end
+    """
+    And a file named "ids.csv" with:
+    """
+    id
+    1
+    2
+    3
+    """
+    When I run `forge -Udata_file=ids.csv -Uid=5 command_script.rb`
+    Then the exit status should be 0
+    And the file "ids.csv" should contain exactly:
+    """
+    id
+    5
+    5
+    5
+
+    """

data/features/deduplication.feature

@@ -0,0 +1,49 @@
+Feature: Deduplicating data in a file
+
+  The `deduplicate` keyword can be used to remove duplicate records from a file. Only the first occurence of each
+  duplicate record is kept.
+
+  Parameters:
+  - The source file to be deduplicated. This parameter is mandatory.
+  - into: The target file name. If no target file is specified then the source file is overwritten with the result
+  of the deduplication.
+  - using: The fields to be used to determine whether or not a record is a duplicate. If not specified then all
+  fields of the source file are used.
+
+
+  Scenario: Single file transformation
+    Given a file named "command_script.rb" with:
+    """
+    file :items do
+      field :rownum
+      field :item_id
+      field :item_name
+    end
+
+    file :unique_items do
+      field :item_id
+      field :item_name
+    end
+
+    deduplicate :items, into: :unique_items, using: :item_id
+    """
+    And a file named "items.csv" with:
+    """
+    rownum,item_id,item_name
+    1,Item1,Item name 1
+    2,Item1,Item name 1
+    3,Item2,Item name 2
+    4,Item2,Item name 2
+    5,Item3,Item name 3
+    """
+    When I run `forge command_script.rb`
+    Then the exit status should be 0
+    And a file named "unique_items.csv" should exist
+    And the file "unique_items.csv" should contain exactly:
+    """
+    item_id,item_name
+    Item1,Item name 1
+    Item2,Item name 2
+    Item3,Item name 3
+
+    """

data/features/file/file_format_options.feature

@@ -0,0 +1,146 @@
+Feature: Formatting options for CSV files
+
+  The `file` block offers options for setting the CSV file's delimiter character, quote character and encoding.
+  All native Ruby encodings are supported.
+
+  The default values are:
+  - delimiter: ,
+  - quote character: "
+  - encoding: UTF-8
+
+
+  Scenario: Using the default delimiter and quote character
+    Given a file named "command_script.rb" with:
+    """
+    file :items do
+      field :id
+      field :name
+      field :category
+    end
+
+    file :items_copy do
+      field :id
+      field :name
+      field :category
+    end
+
+    transform :items, into: :items_copy do |record|
+      output record
+    end
+    """
+    And a file named "items.csv" with:
+    """
+    id,name,category
+    1,"27"" screen",Screens
+    """
+    When I run `forge command_script.rb`
+    Then the exit status should be 0
+    And a file named "items_copy.csv" should exist
+    And the file "items_copy.csv" should contain exactly:
+    """
+    id,name,category
+    1,"27"" screen",Screens
+
+    """
+
+
+  Scenario: Using custom delimiter and quote character
+    Given a file named "command_script.rb" with:
+    """
+    file :items do
+      delimiter ";"
+      quote "'"
+
+      field :id
+      field :name
+      field :category
+    end
+
+    file :items_copy do
+      delimiter "|"
+      quote "`"
+
+      field :id
+      field :name
+      field :category
+    end
+
+    transform :items, into: :items_copy do |record|
+      output record
+    end
+    """
+    And a file named "items.csv" with:
+    """
+    id;name;category
+    1;'27'' screen';Screens
+    2;24` screen;Screens
+    """
+    When I run `forge command_script.rb`
+    Then the exit status should be 0
+    And a file named "items_copy.csv" should exist
+    And the file "items_copy.csv" should contain exactly:
+    """
+    id|name|category
+    1|27' screen|Screens
+    2|`24`` screen`|Screens
+
+    """
+
+
+  Scenario: Using custom encoding
+    Given a file named "command_script.rb" with:
+    """
+    file :items do
+      encoding "UTF-8"
+
+      field :id
+      field :name
+      field :category
+    end
+
+    file :items_latin2 do
+      encoding "ISO-8859-2"
+
+      field :id
+      field :name
+      field :category
+    end
+
+    file :items_copy do
+      encoding "UTF-8"
+
+      field :id
+      field :name
+      field :category
+    end
+
+    transform :items, into: :items_latin2 do |record|
+      output record
+    end
+
+    transform :items_latin2, into: :items_copy do |record|
+      output record
+    end
+    """
+    And a file named "items.csv" with:
+    """
+    id,name,category
+    1,Képernyő,Screens
+    """
+    When I run `forge command_script.rb`
+    Then the exit status should be 0
+    And the following files should exist:
+      | items_latin2.csv |
+      | items_copy.csv   |
+    And the file "items_latin2.csv" should contain in "ISO-8859-2" encoding exactly:
+    """
+    id,name,category
+    1,Képernyő,Screens
+
+    """
+    And the file "items_copy.csv" should contain in "UTF-8" encoding exactly:
+    """
+    id,name,category
+    1,Képernyő,Screens
+
+    """

data/features/file/has_header_row.feature

@@ -0,0 +1,62 @@
+Feature: File `has_header` option
+
+  The `has_header_row` option of a `file` block specifies whether or not the corresponding CSV file has a header row.
+  If a file is specified to have a header row then the first row of the file is skipped during transformation.
+  If not, then all rows of the file are processed. When writing into files with a header row, the fields specified
+  for that file are written as the header of the CSV file. If an output file is specified to have no header then
+  only data rows are written into it.
+
+
+  Scenario: Using the has_header option
+    Given a file named "command_script.rb" with:
+    """
+    file :items do
+      has_header_row true
+
+      field :id
+      field :name
+    end
+
+    file :items_without_header do
+      has_header_row false
+
+      field :id
+      field :name
+    end
+
+    file :items_copy do
+      has_header_row true
+
+      field :id
+      field :name
+    end
+
+    transform :items, into: :items_without_header do |record|
+      output record
+    end
+
+    transform :items_without_header, into: :items_copy do |record|
+      output record
+    end
+    """
+    And a file named "items.csv" with:
+    """
+    id,name
+    1,data
+    """
+    When I run `forge command_script.rb`
+    Then the exit status should be 0
+    And the following files should exist:
+      | items_without_header.csv |
+      | items_copy.csv           |
+    And the file "items_without_header.csv" should contain exactly:
+    """
+    1,data
+
+    """
+    And the file "items_copy.csv" should contain exactly:
+    """
+    id,name
+    1,data
+
+    """

data/features/step_definitions/file_steps.rb

@@ -0,0 +1,8 @@
+Then /^the file "([^"]*)" should be empty$/ do |file_name|
+  step %Q(the file "#{file_name}" should contain exactly:), ""
+end
+
+
+Then /^the file "([^"]*)" should contain in "([^"]*)" encoding exactly:$/ do |file_name, encoding, content|
+  prep_for_fs_check { expect(IO.read(file_name, encoding: encoding).encode("UTF-8", encoding)).to eq content }
+end

data/features/support/env.rb

@@ -0,0 +1,8 @@
+require 'aruba'
+require 'aruba/in_process'
+require 'aruba/cucumber'
+
+require 'data_forge'
+
+Aruba::InProcess.main_class = DataForge::CLI::Main
+Aruba.process = Aruba::InProcess

data/features/transform/output_command.feature

@@ -0,0 +1,123 @@
+Feature: Using the `output` command
+
+  The `output` command in a transformation block writes the specified data (HasH) into the output file.
+  Omitting this command conditionally for a row will skip the record and omitting it unconditionally will
+  simply create an empty file without outputting any data into it.
+
+
+  Background:
+    Given a file named "items.csv" with:
+    """
+    id
+    1
+    2
+    3
+    """
+
+
+  Scenario: Outputting arbitrary data as Hash
+    Given a file named "command_script.rb" with:
+    """
+    file :items do
+      field :id
+    end
+
+    file :books do
+      field :author
+      field :title
+    end
+
+    transform :items, into: :books do |record|
+      output title: "Title #{record[:id]}",
+             author: "Author #{record[:id]}"
+    end
+    """
+    When I run `forge command_script.rb`
+    Then the exit status should be 0
+    And a file named "books.csv" should exist
+    And the file "books.csv" should contain exactly:
+    """
+    author,title
+    Author 1,Title 1
+    Author 2,Title 2
+    Author 3,Title 3
+
+    """
+
+
+  Scenario: No output in transform block creates empty file
+    Given a file named "command_script.rb" with:
+    """
+    file :items do
+      field :id
+    end
+
+    file :items_empty do
+      field :id
+    end
+
+    transform :items, into: :items_empty do end
+    """
+    When I run `forge command_script.rb`
+    Then the exit status should be 0
+    And a file named "items_empty.csv" should exist
+    And the file "items_empty.csv" should be empty
+
+
+  Scenario: Omitting output conditionally skips current record
+    Given a file named "command_script.rb" with:
+    """
+    file :items do
+      field :id
+    end
+
+    file :items_missing do
+      field :id
+    end
+
+    transform :items, into: :items_missing do |record|
+      output record unless "2" == record[:id]
+    end
+    """
+    When I run `forge command_script.rb`
+    Then the exit status should be 0
+    And a file named "items_missing.csv" should exist
+    And the file "items_missing.csv" should contain exactly:
+    """
+    id
+    1
+    3
+
+    """
+
+
+  Scenario: Multiple output commands multiply lines
+    Given a file named "command_script.rb" with:
+    """
+    file :items do
+      field :id
+    end
+
+    file :items_doubled do
+      field :id
+    end
+
+    transform :items, into: :items_doubled do |record|
+      output record
+      output record
+    end
+    """
+    When I run `forge command_script.rb`
+    Then the exit status should be 0
+    And a file named "items_doubled.csv" should exist
+    And the file "items_doubled.csv" should contain exactly:
+    """
+    id
+    1
+    1
+    2
+    2
+    3
+    3
+
+    """

data/features/transform/outputting_to_multiple_files.feature

@@ -0,0 +1,57 @@
+Feature: Splitting a file into multiple files
+
+  The `transform` block can take multiple output file parameters, in which case the `output` command can be used
+  to specify which file to write to.
+
+
+  Scenario:
+    Given a file named "command_script.rb" with:
+    """
+    file :items do
+      field :name
+    end
+
+    file :items_not_a do
+      field :name
+    end
+
+    file :items_not_b do
+      field :name
+    end
+
+    transform :items, into: [:items_not_a, :items_not_b] do |record|
+      if record[:name].start_with? "a"
+        output record, to: :items_not_b
+      elsif record[:name].start_with? "b"
+        output record, to: :items_not_a
+      else
+        output record, to: [:items_not_a, :items_not_b]
+      end
+    end
+    """
+    And a file named "items.csv" with:
+    """
+    name
+    a
+    b
+    c
+    """
+    When I run `forge command_script.rb`
+    Then the exit status should be 0
+    And the following files should exist:
+      | items_not_a.csv |
+      | items_not_b.csv |
+    And the file "items_not_a.csv" should contain exactly:
+    """
+    name
+    b
+    c
+
+    """
+    And the file "items_not_b.csv" should contain exactly:
+    """
+    name
+    a
+    c
+
+    """