data_forge 0.1 → 0.1.1

data/CHANGELOG.md

@@ -0,0 +1,11 @@
+# DataForge Changelog
+
+In reverse chronological order:
+
+## 0.1.1
+
+* Added file definition inheritance using the `file :second, like: :first do … end` directive.
+
+## 0.1
+
+Initial release with basic CSV file transformation functionality.

data/data_forge.gemspec

@@ -18,9 +18,11 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.required_ruby_version = '>= 1.9'
+
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "cucumber"
-  spec.add_development_dependency "aruba"
+  spec.add_development_dependency "cucumber", "~> 1.3.15"
+  spec.add_development_dependency "aruba", "~> 0.5.4"
 end

data/features/file/definition_inheritance.feature

@@ -0,0 +1,76 @@
+Feature: File definition inheritance
+
+  File definitions can be "inherited" using the `like` directive to the `file` command. The inherited structure
+  can be further customized with the initialization block supplied to the `file` command.
+
+
+  Scenario: Using the same definition
+    Given a file named "command_script.rb" with:
+    """
+    file :items do
+      field :id
+      field :name
+    end
+
+    file :items_copy, like: :items
+
+    transform :items, into: :items_copy do |record|
+      output record
+    end
+    """
+    And a file named "items.csv" with:
+    """
+    id,name
+    Item1,Item name 1
+    Item2,Item name 2
+    Item3,Item name 3
+    """
+    When I run `forge command_script.rb`
+    Then the exit status should be 0
+    And a file named "items_copy.csv" should exist
+    And the file "items_copy.csv" should contain exactly:
+    """
+    id,name
+    Item1,Item name 1
+    Item2,Item name 2
+    Item3,Item name 3
+
+    """
+
+
+  Scenario: Customizing inherited definition
+    Given a file named "command_script.rb" with:
+    """
+    file :items do
+      field :id
+      field :name
+    end
+
+    file :items_copy, like: :items do
+      field :comment
+      without_field :id
+    end
+
+    transform :items, into: :items_copy do |record|
+      record[:comment] = "Just a comment"
+      output record
+    end
+    """
+    And a file named "items.csv" with:
+    """
+    id,name
+    Item1,Item name 1
+    Item2,Item name 2
+    Item3,Item name 3
+    """
+    When I run `forge command_script.rb`
+    Then the exit status should be 0
+    And a file named "items_copy.csv" should exist
+    And the file "items_copy.csv" should contain exactly:
+    """
+    name,comment
+    Item name 1,Just a comment
+    Item name 2,Just a comment
+    Item name 3,Just a comment
+
+    """

data/lib/data_forge/dsl/commands.rb

@@ -2,8 +2,8 @@ module DataForge
   module DSL
     module Commands
 
-      def file(name, &initialization_block)
-        File.register_file_definition name, &initialization_block
+      def file(name, options = {}, &initialization_block)
+        File.register_file_definition name, options, &initialization_block
       end
 
 

data/lib/data_forge/file.rb

@@ -11,24 +11,38 @@ module DataForge
 
     class << self
 
-      def register_file_definition(name, &initialization_block)
-        @file_definitions[name] = File::RecordFileDefinition.from_input name, &initialization_block
+      attr_reader :file_definitions
+
+
+
+      def register_file_definition(name, options, &initialization_block)
+        @file_definitions[name] = if options[:like]
+                                    File::RecordFileDefinition.from_copy definition(options[:like]), name, &initialization_block
+                                  else
+                                    File::RecordFileDefinition.from_input name, &initialization_block
+                                  end
       end
 
 
 
       def reader_for(definition_name)
-        raise "Unknown file reference '#{definition_name}'" unless @file_definitions.has_key? definition_name
-
-        RecordFileReader.for @file_definitions[definition_name]
+        RecordFileReader.for definition definition_name
       end
 
 
 
       def writer_for(definition_name)
-        raise "Unknown file reference '#{definition_name}'" unless @file_definitions.has_key? definition_name
+        RecordFileWriter.for definition definition_name
+      end
+
+
+
+      private
+
+      def definition(name)
+        raise "Unknown file reference '#{name}'" unless file_definitions.has_key? name
 
-        RecordFileWriter.for @file_definitions[definition_name]
+        file_definitions[name]
       end
 
     end

data/lib/data_forge/file/csv/csv_record_file_definition.rb

@@ -36,10 +36,27 @@ module DataForge
 
 
 
+        def without_field(name)
+          @fields.delete name
+        end
+
+
+
         def field_names
           @fields.keys
         end
 
+
+
+        def copy(definition)
+          delimiter definition.delimiter
+          quote definition.quote
+          encoding definition.encoding
+          has_header_row definition.has_header_row
+
+          definition.fields.each { |name, type| field name, type }
+        end
+
       end
     end
   end

data/lib/data_forge/file/record_file_definition.rb

@@ -3,7 +3,16 @@ module DataForge
     module RecordFileDefinition
 
       def self.from_input(name, &initialization_block)
-        CSV::CSVRecordFileDefinition.new(name).tap { |definition| definition.instance_eval &initialization_block }
+        from_copy nil, name, &initialization_block
+      end
+
+
+
+      def self.from_copy(source_definition, name, &initialization_block)
+        CSV::CSVRecordFileDefinition.new(name).tap do |definition|
+          definition.copy source_definition if source_definition
+          definition.instance_eval &initialization_block if initialization_block
+        end
       end
 
 
@@ -12,6 +21,11 @@ module DataForge
 
       attr_reader :name, :fields
 
+
+
+      def copy(definition)
+      end
+
     end
   end
 end

data/lib/data_forge/transformation.rb

@@ -3,8 +3,8 @@ module DataForge
 
     autoload :Deduplication, 'data_forge/transformation/deduplication'
     autoload :RubyTransformation, 'data_forge/transformation/ruby_transformation'
-    autoload :TransformationBase, 'data_forge/transformation/transformation_base'
     autoload :RubyTransformationContext, 'data_forge/transformation/ruby_transformation_context'
+    autoload :TransformationBase, 'data_forge/transformation/transformation_base'
 
   end
 end

data/lib/data_forge/version.rb

@@ -1,3 +1,3 @@
 module DataForge
-  VERSION = "0.1"
+  VERSION = "0.1.1"
 end

data/spec/data_forge/dsl/commands_spec.rb

@@ -7,10 +7,17 @@ describe DataForge::DSL::Commands do
 
   describe "#file" do
     it "should register a file descriptor" do
-      expect(DataForge::File).to receive(:register_file_definition).with(:name) { |&blk| expect(blk).to be block }
+      expect(DataForge::File).to receive(:register_file_definition).with(:name, {}) { |&blk| expect(blk).to be block }
 
       dsl_object.file :name, &block
     end
+
+    it "should pass along any options received" do
+      options = {like: :other_definition}
+      expect(DataForge::File).to receive(:register_file_definition).with(:name, options) { |&blk| expect(blk).to be block }
+
+      dsl_object.file :name, options, &block
+    end
   end
 
 

data/spec/data_forge/file/csv/csv_record_file_definition_spec.rb

@@ -28,6 +28,18 @@ describe DataForge::File::CSV::CSVRecordFileDefinition do
   end
 
 
+  describe "#without_field" do
+    it "should remove a registered field" do
+      subject.field :field1
+      subject.field :field2
+
+      subject.without_field :field1
+
+      expect(subject.fields).to eq(field2: String)
+    end
+  end
+
+
   describe "#fields" do
     it "should return an empty Hash if no fields are defined" do
       expect(subject.fields).to eq({})
@@ -94,4 +106,26 @@ describe DataForge::File::CSV::CSVRecordFileDefinition do
     end
   end
 
+
+  describe "#copy" do
+    it "should copy the non-identifying attributes of the specified definition" do
+      definition = described_class.new :def1
+      definition.file_name "first.csv"
+      definition.delimiter ";"
+      definition.quote "'"
+      definition.encoding "Latin2"
+      definition.field :f1
+      definition.field :f2
+
+      subject.copy definition
+
+      expect(subject.name).to eq :definition_name
+      expect(subject.file_name).to eq "definition_name.csv"
+      expect(subject.delimiter).to eq ";"
+      expect(subject.quote).to eq "'"
+      expect(subject.encoding).to eq "Latin2"
+      expect(subject.field_names).to eq [:f1, :f2]
+    end
+  end
+
 end

data/spec/data_forge/file/record_file_definition_spec.rb

@@ -1,16 +1,40 @@
 require 'spec_helper'
 
 describe DataForge::File::RecordFileDefinition do
+  let(:definition) { instance_double "DataForge::File::CSV::CSVRecordFileDefinition" }
+  let(:initializer_block) { lambda {} }
 
-  describe ".from_input" do
-    it "should instantiate a CSV file definition and initalize it with the initializer block" do
-      definition = instance_double "DataForge::File::CSV::CSVRecordFileDefinition"
-      initializer_block = lambda {}
+  before do
+    allow(DataForge::File::CSV::CSVRecordFileDefinition).to receive(:new).with(:test).and_return definition
+  end
 
+
+  describe ".from_input" do
+    it "should instantiate a CSV file definition with the given name" do
       expect(DataForge::File::CSV::CSVRecordFileDefinition).to receive(:new).with(:test).and_return definition
-      expect(definition).to receive(:instance_eval) { |&block| expect(block).to be initializer_block }
 
-      expect(subject.from_input :test, &initializer_block).to eq definition
+      expect(subject.from_input :test).to eq definition
+    end
+
+
+    context "when there is an initializer block" do
+      it "should instantiate a CSV file definition and initalize it with the initializer block" do
+        expect(definition).to receive(:instance_eval) { |&block| expect(block).to be initializer_block }
+
+        expect(subject.from_input :test, &initializer_block).to eq definition
+      end
+    end
+  end
+
+
+  describe ".from_copy" do
+    it "should copy the specified file definition and initialize it" do
+      source_definition = instance_double "DataForge::File::CSV::CSVRecordFileDefinition"
+
+      expect(definition).to receive(:copy).with(source_definition).ordered
+      expect(definition).to receive(:instance_eval) { |&block| expect(block).to be initializer_block }.ordered
+
+      expect(subject.from_copy source_definition, :test, &initializer_block).to eq definition
     end
   end
 

data/spec/data_forge/file_spec.rb

@@ -4,20 +4,56 @@ describe DataForge::File do
 
   let(:definition) { instance_double "DataForge::File::RecordFileDefinition" }
 
-  before do
-    allow(DataForge::File::RecordFileDefinition).to receive(:from_input).with(:definition_name).and_return definition
-  end
-
   after do
     subject.instance_variable_set :@file_definitions, {}
   end
 
 
+  describe ".register_file_definition" do
+    let(:initializer_block) { lambda {} }
+
+    context "without any options" do
+      it "should register a named file definition with the specified initializer block" do
+        expect(DataForge::File::RecordFileDefinition).to receive(:from_input).
+                                                           with(:definition_name) { |&block| expect(block).to eq initializer_block }.
+                                                           and_return definition
+
+        subject.register_file_definition :definition_name, {}, &initializer_block
+
+        expect(subject.file_definitions[:definition_name]).to eq definition
+      end
+    end
+
+
+    context "with the :like option" do
+      it "should copy the specified file definition" do
+        subject.register_file_definition :source_definition, {}
+
+        expect(DataForge::File::RecordFileDefinition).to receive(:from_copy).
+                                                           with(subject.file_definitions[:source_definition], :definition_name) { |&block| expect(block).to eq initializer_block }.
+                                                           and_return definition
+
+        subject.register_file_definition :definition_name, like: :source_definition, &initializer_block
+
+        expect(subject.file_definitions[:definition_name]).to eq definition
+      end
+    end
+
+    it "should raise an error if an unknown definition is specified as source" do
+      expect { subject.register_file_definition :def2, like: :def1 }.to raise_error "Unknown file reference 'def1'"
+    end
+  end
+
+
   describe ".reader_for" do
     let(:reader) { double "Reader" }
 
+    before do
+      allow(DataForge::File::RecordFileDefinition).to receive(:from_input).with(:definition_name).and_return definition
+    end
+
     it "should return a record reader for the file with the specified name" do
-      subject.register_file_definition :definition_name
+      subject.register_file_definition :definition_name, {}
 
       expect(DataForge::File::RecordFileReader).to receive(:for).with(definition).and_return reader
 
@@ -33,8 +69,12 @@ describe DataForge::File do
   describe ".writer_for" do
     let(:writer) { double "Writer" }
 
+    before do
+      allow(DataForge::File::RecordFileDefinition).to receive(:from_input).with(:definition_name).and_return definition
+    end
+
     it "should return a record writer for the file with the specified name" do
-      subject.register_file_definition :definition_name
+      subject.register_file_definition :definition_name, {}
 
       expect(DataForge::File::RecordFileWriter).to receive(:for).with(definition).and_return writer
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_forge
 version: !ruby/object:Gem::Version
-  version: '0.1'
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-01 00:00:00.000000000 Z
+date: 2014-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -64,33 +64,33 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.15
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.15
 - !ruby/object:Gem::Dependency
   name: aruba
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.5.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.5.4
 description: DataForge is a data manipulation tool for transferring (and transforming)
   data between flat files and databases.
 email:
@@ -103,6 +103,7 @@ files:
 - .gitignore
 - .rspec
 - .travis.yml
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md
@@ -112,8 +113,9 @@ files:
 - data_forge.gemspec
 - features/accessing_command_line_parameters.feature
 - features/deduplication.feature
-- features/file/file_format_options.feature
-- features/file/has_header_row.feature
+- features/file/csv/file_format_options.feature
+- features/file/csv/has_header_row.feature
+- features/file/definition_inheritance.feature
 - features/step_definitions/file_steps.rb
 - features/support/env.rb
 - features/transform/output_command.feature
@@ -173,7 +175,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '1.9'
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -189,8 +191,9 @@ summary: Pure Ruby ETL and data manipulation tool.
 test_files:
 - features/accessing_command_line_parameters.feature
 - features/deduplication.feature
-- features/file/file_format_options.feature
-- features/file/has_header_row.feature
+- features/file/csv/file_format_options.feature
+- features/file/csv/has_header_row.feature
+- features/file/definition_inheritance.feature
 - features/step_definitions/file_steps.rb
 - features/support/env.rb
 - features/transform/output_command.feature