data_forge 0.1

data/spec/data_forge/transformation/deduplication_spec.rb

@@ -0,0 +1,77 @@
+require 'spec_helper'
+
+describe DataForge::Transformation::Deduplication do
+
+  describe ".from_input" do
+    let(:deduplication) { instance_double described_class.name }
+    let(:reader) { instance_double "DataForge::File::RecordFileReader", name: :source, fields: [:field1, :field2, :field3] }
+    let(:source_writer) { instance_double "DataForge::File::RecordFileWriter" }
+    let(:target_writer) { instance_double "DataForge::File::RecordFileWriter" }
+
+    before do
+      allow(DataForge::File).to receive(:reader_for).with(:source).and_return reader
+      allow(DataForge::File).to receive(:writer_for).with(:source).and_return source_writer
+      allow(DataForge::File).to receive(:writer_for).with(:target).and_return target_writer
+    end
+
+
+    context "when only the source is specified" do
+      it "should return a Deduplication with a writer for the same file" do
+        allow(described_class).to receive(:new).with(reader, source_writer, anything).and_return deduplication
+
+        expect(described_class.from_input :source).to eq deduplication
+      end
+
+      it "should return a Deduplication using all the fields of the source" do
+        allow(described_class).to receive(:new).with(reader, anything, [:field1, :field2, :field3]).and_return deduplication
+
+        expect(described_class.from_input :source).to eq deduplication
+      end
+    end
+
+
+    context "when a target file is specified" do
+      it "should return a Deduplication with the specified writer" do
+        allow(described_class).to receive(:new).with(reader, target_writer, anything).and_return deduplication
+
+        expect(described_class.from_input :source, into: :target).to eq deduplication
+      end
+    end
+
+
+    context "when the unique fields are specified" do
+      it "should return a Deduplication using the specified field, if there is only one" do
+        allow(described_class).to receive(:new).with(anything, anything, [:field1]).and_return deduplication
+
+        expect(described_class.from_input :source, using: :field1).to eq deduplication
+      end
+
+      it "should return a Deduplication using all specified fields, if there is more than one" do
+        allow(described_class).to receive(:new).with(anything, anything, [:field1, :field2]).and_return deduplication
+
+        expect(described_class.from_input :source, using: [:field1, :field2]).to eq deduplication
+      end
+    end
+  end
+
+
+  describe "#execute" do
+    subject { described_class.new reader, writer, [:f1, :f2] }
+
+    let(:writer) { mock_writer }
+    let(:reader) { stub_reader_with_records [{f1: "a", f2: "b", f3: "c"},
+                                             {f1: "a", f2: "b", f3: "c"},
+                                             {f1: "a", f2: "b", f3: "d"},
+                                             {f1: "a", f2: "e", f3: "c"},
+                                             {f1: "f", f2: "b", f3: "c"}] }
+
+    it "should write only the first instance of each source record to the writer" do
+      expect(writer).to receive(:write).with(f1: "a", f2: "b", f3: "c").once
+      expect(writer).to receive(:write).with(f1: "a", f2: "e", f3: "c")
+      expect(writer).to receive(:write).with(f1: "f", f2: "b", f3: "c")
+
+      subject.execute
+    end
+  end
+
+end

data/spec/data_forge/transformation/ruby_transformation_context_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper'
+
+describe DataForge::Transformation::RubyTransformationContext do
+
+  describe "#output" do
+    let(:record) { double "Record" }
+    let(:writer1) { instance_double "DataForge::File::RecordFileWriter", name: :writer1 }
+    let(:writer2) { instance_double "DataForge::File::RecordFileWriter", name: :writer2 }
+
+    context "when the :to directive is not used" do
+      it "should write the record into the record writer" do
+        subject = described_class.new [writer1]
+
+        expect(writer1).to receive(:write).with(record)
+
+        subject.output record
+      end
+
+      it "should raise an error if there is more than 1 writer available" do
+        subject = described_class.new [writer1, writer2]
+
+        expect { subject.output record }.to raise_error "Missing :to directive for `output` command in multiple file transformation"
+      end
+    end
+
+
+    context "when the :to directive is used" do
+      subject { described_class.new [writer1, writer2] }
+
+      it "should write the record into the specified writer, if there is only one" do
+        expect(writer1).to receive(:write).with(record)
+
+        subject.output record, to: :writer1
+      end
+
+      it "should write the record into all specified writers, if there is more than one" do
+        expect(writer1).to receive(:write).with(record)
+        expect(writer2).to receive(:write).with(record)
+
+        subject.output record, to: [:writer1, :writer2]
+      end
+
+      it "should raise an error if an unrecognized target file is specified" do
+        expect { subject.output record, to: :no_such_file }.to raise_error "Unknown target file 'no_such_file' for `output` command"
+      end
+    end
+  end
+
+end

data/spec/data_forge/transformation/ruby_transformation_spec.rb

@@ -0,0 +1,71 @@
+require 'spec_helper'
+
+describe DataForge::Transformation::RubyTransformation do
+  let(:transformation_block) { lambda {} }
+
+  describe ".from_input" do
+    let(:transformation) { instance_double described_class.name }
+    let(:reader) { instance_double "DataForge::File::RecordFileReader", name: :source, fields: [:field1, :field2, :field3] }
+    let(:source_writer) { instance_double "DataForge::File::RecordFileWriter" }
+    let(:target_writer) { instance_double "DataForge::File::RecordFileWriter" }
+    let(:other_target_writer) { instance_double "DataForge::File::RecordFileWriter" }
+
+    before do
+      allow(DataForge::File).to receive(:reader_for).with(:source).and_return reader
+      allow(DataForge::File).to receive(:writer_for).with(:source).and_return source_writer
+      allow(DataForge::File).to receive(:writer_for).with(:target).and_return target_writer
+      allow(DataForge::File).to receive(:writer_for).with(:other_target).and_return other_target_writer
+    end
+
+
+    context "when only the source is specified" do
+      it "should return a RubyTransformation with a single writer for the same file" do
+        allow(described_class).to receive(:new)
+                                  .with(reader, [source_writer]) { |&block| expect(block).to eq transformation_block }
+                                  .and_return transformation
+
+        expect(described_class.from_input :source, &transformation_block).to eq transformation
+      end
+    end
+
+
+    context "when a single writer is specified" do
+      it "should return a RubyTransformation with the specified writer" do
+        allow(described_class).to receive(:new).with(reader, [target_writer]).and_return transformation
+
+        expect(described_class.from_input :source, into: :target, &transformation_block).to eq transformation
+      end
+    end
+
+
+    context "when multiple writers are specified" do
+      it "should return a RubyTransformation with all specified writers" do
+        allow(described_class).to receive(:new).with(reader, [target_writer, other_target_writer]).and_return transformation
+
+        expect(described_class.from_input :source, into: [:target, :other_target], &transformation_block).to eq transformation
+      end
+    end
+  end
+
+
+  describe "#execute" do
+    subject { described_class.new reader, writers, &transformation_block }
+
+    let(:writers) { [mock_writer] }
+    let(:reader) { stub_reader_with_records [{f1: "a", f2: "b"},
+                                             {f1: "c", f2: "d"},
+                                             {f1: "e", f2: "f"}] }
+
+    it "should write only the first instance of each source record to the writer" do
+      context = instance_double "DataForge::Transformation::RubyTransformationContext"
+      expect(DataForge::Transformation::RubyTransformationContext).to receive(:new).with(writers).and_return context
+
+      expect(context).to receive(:instance_exec).with(f1: "a", f2: "b") { |&block| expect(block).to eq transformation_block }
+      expect(context).to receive(:instance_exec).with(f1: "c", f2: "d") { |&block| expect(block).to eq transformation_block }
+      expect(context).to receive(:instance_exec).with(f1: "e", f2: "f") { |&block| expect(block).to eq transformation_block }
+
+      subject.execute
+    end
+  end
+
+end

data/spec/data_forge_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+describe DataForge do
+
+  it "should have a version number" do
+    expect(DataForge::VERSION).not_to be_nil
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'data_forge'
+require_relative 'support/helpers/record_reader_helper'
+require_relative 'support/helpers/record_writer_helper'
+
+
+RSpec.configure do |config|
+  config.order = "random"
+  config.raise_errors_for_deprecations!
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_doubled_constant_names = true
+  end
+
+  config.include RSpec::Helpers::RecordReaderHelper
+  config.include RSpec::Helpers::RecordWriterHelper
+end

data/spec/support/helpers/record_reader_helper.rb

@@ -0,0 +1,17 @@
+module RSpec
+  module Helpers
+    module RecordReaderHelper
+
+      def stub_reader_with_records(records)
+        reader = instance_double "DataForge::File::RecordFileReader"
+
+        match_records = receive(:each_record)
+        records.each { |record| match_records.and_yield record }
+        allow(reader).to match_records
+
+        reader
+      end
+
+    end
+  end
+end

data/spec/support/helpers/record_writer_helper.rb

@@ -0,0 +1,16 @@
+module RSpec
+  module Helpers
+    module RecordWriterHelper
+
+      def mock_writer
+        writer = instance_double "DataForge::File::RecordFileWriter"
+
+        expect(writer).to receive(:open)
+        expect(writer).to receive(:close)
+
+        writer
+      end
+
+    end
+  end
+end

metadata

@@ -0,0 +1,218 @@
+--- !ruby/object:Gem::Specification
+name: data_forge
+version: !ruby/object:Gem::Version
+  version: '0.1'
+  prerelease: 
+platform: ruby
+authors:
+- Zoltan Ormandi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aruba
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: DataForge is a data manipulation tool for transferring (and transforming)
+  data between flat files and databases.
+email:
+- zoltan.ormandi@gmail.com
+executables:
+- forge
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/forge
+- config/cucumber.yml
+- data_forge.gemspec
+- features/accessing_command_line_parameters.feature
+- features/deduplication.feature
+- features/file/file_format_options.feature
+- features/file/has_header_row.feature
+- features/step_definitions/file_steps.rb
+- features/support/env.rb
+- features/transform/output_command.feature
+- features/transform/outputting_to_multiple_files.feature
+- features/transform/overwrite_original_file.feature
+- features/transform/record_transformation.feature
+- lib/data_forge.rb
+- lib/data_forge/cli.rb
+- lib/data_forge/cli/main.rb
+- lib/data_forge/cli/options.rb
+- lib/data_forge/dsl.rb
+- lib/data_forge/dsl/attributes.rb
+- lib/data_forge/dsl/commands.rb
+- lib/data_forge/dsl/helpers.rb
+- lib/data_forge/file.rb
+- lib/data_forge/file/csv.rb
+- lib/data_forge/file/csv/csv_record_file_definition.rb
+- lib/data_forge/file/csv/csv_record_file_reader.rb
+- lib/data_forge/file/csv/csv_record_file_writer.rb
+- lib/data_forge/file/record_file_definition.rb
+- lib/data_forge/file/record_file_reader.rb
+- lib/data_forge/file/record_file_writer.rb
+- lib/data_forge/transformation.rb
+- lib/data_forge/transformation/deduplication.rb
+- lib/data_forge/transformation/ruby_transformation.rb
+- lib/data_forge/transformation/ruby_transformation_context.rb
+- lib/data_forge/transformation/transformation_base.rb
+- lib/data_forge/version.rb
+- spec/data_forge/cli/main_spec.rb
+- spec/data_forge/cli/options_spec.rb
+- spec/data_forge/cli_spec.rb
+- spec/data_forge/dsl/commands_spec.rb
+- spec/data_forge/dsl/helpers_spec.rb
+- spec/data_forge/file/csv/csv_record_file_definition_spec.rb
+- spec/data_forge/file/csv/csv_record_file_reader_spec.rb
+- spec/data_forge/file/csv/csv_record_file_writer_spec.rb
+- spec/data_forge/file/record_file_definition_spec.rb
+- spec/data_forge/file/record_file_reader_spec.rb
+- spec/data_forge/file/record_file_writer_spec.rb
+- spec/data_forge/file_spec.rb
+- spec/data_forge/transformation/deduplication_spec.rb
+- spec/data_forge/transformation/ruby_transformation_context_spec.rb
+- spec/data_forge/transformation/ruby_transformation_spec.rb
+- spec/data_forge_spec.rb
+- spec/spec_helper.rb
+- spec/support/helpers/record_reader_helper.rb
+- spec/support/helpers/record_writer_helper.rb
+homepage: https://github.com/zormandi/data_forge
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23.2
+signing_key: 
+specification_version: 3
+summary: Pure Ruby ETL and data manipulation tool.
+test_files:
+- features/accessing_command_line_parameters.feature
+- features/deduplication.feature
+- features/file/file_format_options.feature
+- features/file/has_header_row.feature
+- features/step_definitions/file_steps.rb
+- features/support/env.rb
+- features/transform/output_command.feature
+- features/transform/outputting_to_multiple_files.feature
+- features/transform/overwrite_original_file.feature
+- features/transform/record_transformation.feature
+- spec/data_forge/cli/main_spec.rb
+- spec/data_forge/cli/options_spec.rb
+- spec/data_forge/cli_spec.rb
+- spec/data_forge/dsl/commands_spec.rb
+- spec/data_forge/dsl/helpers_spec.rb
+- spec/data_forge/file/csv/csv_record_file_definition_spec.rb
+- spec/data_forge/file/csv/csv_record_file_reader_spec.rb
+- spec/data_forge/file/csv/csv_record_file_writer_spec.rb
+- spec/data_forge/file/record_file_definition_spec.rb
+- spec/data_forge/file/record_file_reader_spec.rb
+- spec/data_forge/file/record_file_writer_spec.rb
+- spec/data_forge/file_spec.rb
+- spec/data_forge/transformation/deduplication_spec.rb
+- spec/data_forge/transformation/ruby_transformation_context_spec.rb
+- spec/data_forge/transformation/ruby_transformation_spec.rb
+- spec/data_forge_spec.rb
+- spec/spec_helper.rb
+- spec/support/helpers/record_reader_helper.rb
+- spec/support/helpers/record_writer_helper.rb