data_forge 0.1

data/lib/data_forge.rb

@@ -0,0 +1,13 @@
+require 'data_forge/version'
+
+module DataForge
+
+  autoload :CLI, 'data_forge/cli'
+  autoload :DSL, 'data_forge/dsl'
+  autoload :File, 'data_forge/file'
+  autoload :Transformation, 'data_forge/transformation'
+
+end
+
+self.extend DataForge::DSL::Commands,
+            DataForge::DSL::Helpers

data/spec/data_forge/cli/main_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+
+describe DataForge::CLI::Main do
+
+  let(:options) { DataForge::CLI::Options.new }
+  let(:args) { double "ARGV" }
+  let(:stdout) { double "STDOUT" }
+  let(:stderr) { double "STDERR" }
+  let(:kernel) { double "Kernel" }
+
+  subject { described_class.new args, STDIN, stdout, stderr, kernel }
+
+  before do
+    allow(DataForge::CLI).to receive(:parse_options).with(args, stdout).and_return options
+  end
+
+
+  describe "#execute!" do
+    it "should execute the command script specified in the options" do
+      options.command_script = "command_script.rb"
+
+      expect(subject).to receive(:load).with("command_script.rb")
+
+      subject.execute!
+    end
+
+    it "should not execute the command script if the options direct to stop execution" do
+      options.execute = false
+
+      expect(subject).not_to receive(:load)
+
+      subject.execute!
+    end
+
+    it "should output an error message in case of an error" do
+      allow(subject).to receive(:load).and_raise "Error message"
+
+      expect(stderr).to receive(:puts).with "ERROR: Error message"
+      expect(kernel).to receive(:exit).with 1
+
+      subject.execute!
+    end
+  end
+
+end

data/spec/data_forge/cli/options_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+describe DataForge::CLI::Options do
+
+  let(:stdout) { StringIO.new }
+  subject { DataForge::CLI::Options }
+
+  describe ".parse" do
+    context "when parsing the command script parameter" do
+      it "should accept a single command script parameter" do
+        options = subject.parse(%w[command_script.rb])
+
+        expect(options.command_script).to eq "command_script.rb"
+      end
+
+      it "should raise an error if no command script is specified" do
+        expect { subject.parse(%w[]) }.to raise_error "No command script specified"
+      end
+
+      it "should raise an error if there is more than one command script specified" do
+        expect { subject.parse(%w[command_script1.rb command_script2.rb]) }.to raise_error "More than one command script specified"
+      end
+    end
+
+
+    context "when parsing the --help switch" do
+      it "should print the help information" do
+        subject.parse(%w[--help], stdout)
+
+        expect(stdout.string).to include "Usage: [bundle exec] forge [options] command_script.rb"
+      end
+    end
+
+
+    context "when parsing the --version switch" do
+      it "should print the version" do
+        subject.parse(%w[--version], stdout)
+
+        expect(stdout.string).to match /DataForge, version \d+(\.\d+)*/
+      end
+    end
+
+
+    context "when parsing user-defined parameters" do
+      it "should accept a name-value pair as a parameter with the -U switch" do
+        options = subject.parse(%w[-Ucustomer=test command_script.rb])
+
+        expect(options.user_params).to eq(customer: "test")
+      end
+
+      it "should accept multiple user-defined parameters" do
+        options = subject.parse(%w[-Ucustomer=test -Udata_file=items.csv command_script.rb])
+
+        expect(options.user_params).to eq(customer: "test", data_file: "items.csv")
+      end
+    end
+
+
+    it "should raise an error if an unknown option is specified" do
+      expect { subject.parse(%w[--unknown]) }.to raise_error OptionParser::InvalidOption
+    end
+  end
+
+end

data/spec/data_forge/cli_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+describe DataForge::CLI do
+  let(:options) { DataForge::CLI::Options.new }
+  let(:args) { double "ARGV" }
+  let(:stdout) { double "STDOUT" }
+
+  before do
+    allow(DataForge::CLI::Options).to receive(:parse).with(args, stdout).and_return options
+  end
+
+  after do
+    subject.instance_variable_set :@command_script, nil
+    subject.instance_variable_set :@user_params, nil
+  end
+
+
+  describe ".parse_options" do
+    it "should return the command line options parsed into an Options object" do
+      expect(subject.parse_options args, stdout).to eq options
+    end
+  end
+
+
+  describe ".command_script" do
+    it "should be nil by default" do
+      expect(subject.command_script).to be_nil
+    end
+
+    it "should return the command script specified in options that were parsed" do
+      options.command_script = "command_script.rb"
+
+      subject.parse_options args, stdout
+
+      expect(subject.command_script).to eq "command_script.rb"
+    end
+  end
+
+
+  describe ".user_params" do
+    it "should be nil by default" do
+      expect(subject.user_params).to be_nil
+    end
+
+    it "should return the user-defined parameters specified in options that were parsed" do
+      options.user_params = {p1: "v1", p2: "v2"}
+
+      subject.parse_options args, stdout
+
+      expect(subject.user_params).to eq(p1: "v1", p2: "v2")
+    end
+  end
+
+end

data/spec/data_forge/dsl/commands_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+
+describe DataForge::DSL::Commands do
+
+  let(:dsl_object) { Object.new.tap { |object| object.extend DataForge::DSL::Commands } }
+  let(:block) { lambda {} }
+
+  describe "#file" do
+    it "should register a file descriptor" do
+      expect(DataForge::File).to receive(:register_file_definition).with(:name) { |&blk| expect(blk).to be block }
+
+      dsl_object.file :name, &block
+    end
+  end
+
+
+  describe "#transform" do
+    it "should create a file transformation and execute it" do
+      transformation = instance_double "DataForge::Transformation::RubyTransformation"
+
+      allow(DataForge::Transformation::RubyTransformation).to receive(:from_input)
+                                                              .with(:source, into: :target) { |&blk| expect(blk).to be block }
+                                                              .and_return(transformation)
+      expect(transformation).to receive(:execute)
+
+      dsl_object.transform :source, into: :target, &block
+    end
+  end
+
+
+  describe "#deduplicate" do
+    it "should create a deduplication transformation and execute it" do
+      deduplication = instance_double "DataForge::Transformation::Deduplication"
+      allow(DataForge::Transformation::Deduplication).to receive(:from_input).with(:items, into: :unique_items, using: :item_id).and_return(deduplication)
+
+      expect(deduplication).to receive(:execute)
+
+      dsl_object.deduplicate :items, into: :unique_items, using: :item_id
+    end
+  end
+
+end

data/spec/data_forge/dsl/helpers_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+describe DataForge::DSL::Helpers do
+
+  subject { Object.new.extend described_class }
+
+  describe "PARAMS" do
+    it "should return the user parameters passed in through the CLI" do
+      expect(DataForge::CLI).to receive(:user_params).and_return "user defined parameters"
+
+      expect(subject.instance_eval { PARAMS }).to eq "user defined parameters"
+    end
+  end
+
+
+  describe "COMMAND_SCRIPT" do
+    it "should return the command script that is currently executing" do
+      expect(DataForge::CLI).to receive(:command_script).and_return "command_script.rb"
+
+      expect(subject.instance_eval { COMMAND_SCRIPT }).to eq "command_script.rb"
+    end
+  end
+
+end

data/spec/data_forge/file/csv/csv_record_file_definition_spec.rb

@@ -0,0 +1,97 @@
+require 'spec_helper'
+
+describe DataForge::File::CSV::CSVRecordFileDefinition do
+
+  subject { described_class.new :definition_name }
+
+  describe "#name" do
+    it "should return the name of the descriptor" do
+      expect(subject.name).to eq :definition_name
+    end
+  end
+
+
+  describe "#field" do
+    it "should define a field with a type" do
+      subject.field :field1, String
+
+      expect(subject.fields).to eq(field1: String)
+    end
+
+    context "when called without a type" do
+      it "should define a String field" do
+        subject.field :field1
+
+        expect(subject.fields).to eq(field1: String)
+      end
+    end
+  end
+
+
+  describe "#fields" do
+    it "should return an empty Hash if no fields are defined" do
+      expect(subject.fields).to eq({})
+    end
+
+    it "should return the fields and types (as a Hash) that are defined" do
+      subject.field :field1, String
+      subject.field :field2, Fixnum
+
+      expect(subject.fields).to eq(field1: String, field2: Fixnum)
+    end
+  end
+
+
+  describe "#field_names" do
+    it "should return an empty array if no fields are defined" do
+      expect(subject.field_names).to eq []
+    end
+
+    it "should return the names of the fields that are defined" do
+      subject.field :field1, String
+      subject.field :field2, Fixnum
+
+      expect(subject.field_names).to eq [:field1, :field2]
+    end
+  end
+
+
+  describe "attributes" do
+    {file_name: "definition_name.csv",
+     delimiter: ",",
+     quote: '"',
+     encoding: "UTF-8",
+     has_header_row: true}
+    .each do |attribute_name, default_value|
+
+      describe "#{attribute_name}" do
+        it "should return or set the attribute value" do
+          subject.public_send attribute_name, "new value"
+
+          expect(subject.public_send(attribute_name)).to eq "new value"
+        end
+
+        context "when not overridden" do
+          it "should return the default value of the attribute" do
+            expect(subject.send(attribute_name)).to eq default_value
+          end
+        end
+      end
+
+    end
+  end
+
+
+  describe "#separator" do
+    it "should be an alias for #delimiter" do
+      expect(subject.separator).to eq ","
+
+      subject.delimiter ";"
+      expect(subject.separator).to eq ";"
+
+      subject.separator "|"
+      expect(subject.delimiter).to eq "|"
+    end
+  end
+
+end

data/spec/data_forge/file/csv/csv_record_file_reader_spec.rb

@@ -0,0 +1,78 @@
+require 'spec_helper'
+
+describe DataForge::File::CSV::CSVRecordFileReader do
+
+  let(:csv_file) { instance_double "CSV" }
+  let(:definition) { instance_double "DataForge::File::CSV::CSVRecordFileDefinition",
+                                     name: :test,
+                                     file_name: "test.csv",
+                                     field_names: [:field1, :field2],
+                                     delimiter: "delimiter",
+                                     quote: "quote",
+                                     encoding: "encoding",
+                                     has_header_row: true }
+
+  subject { described_class.new definition }
+
+
+  describe "#definition" do
+    it "should return the file definition the writer was created for" do
+      expect(subject.definition).to eq definition
+    end
+  end
+
+
+  describe "#name" do
+    it "should return the file definition's name" do
+      expect(subject.name).to eq :test
+    end
+  end
+
+
+  describe "#fields" do
+    it "should return the file definition's fields" do
+      expect(subject.fields).to eq [:field1, :field2]
+    end
+  end
+
+
+  describe "#each_record" do
+    context "when the CSV file has a header row" do
+      it "should skip the header row and iterate through all records in the CSV file" do
+        expect(CSV).to receive(:open).with("test.csv", {col_sep: "delimiter",
+                                                        quote_char: "quote",
+                                                        encoding: "encoding",
+                                                        return_headers: false}).and_yield csv_file
+        allow(csv_file).to receive(:shift).and_return(["field1", "field2"], [1, 2], [3, 4], nil)
+
+        records = []
+        subject.each_record { |record| records << record }
+
+        expect(records).to eq [{field1: 1, field2: 2}, {field1: 3, field2: 4}]
+      end
+    end
+
+
+    context "when the CSV file has no header row" do
+      let(:definition) { instance_double "DataForge::File::CSV::CSVRecordFileDefinition",
+                                         name: :test,
+                                         file_name: "test.csv",
+                                         field_names: [:field1, :field2],
+                                         delimiter: "delimiter",
+                                         quote: "quote",
+                                         encoding: "encoding",
+                                         has_header_row: false }
+
+      it "should iterate through all records in the CSV file" do
+        allow(CSV).to receive(:open).and_yield csv_file
+        allow(csv_file).to receive(:shift).and_return([1, 2], [3, 4], nil)
+
+        records = []
+        subject.each_record { |record| records << record }
+
+        expect(records).to eq [{field1: 1, field2: 2}, {field1: 3, field2: 4}]
+      end
+    end
+  end
+
+end

data/spec/data_forge/file/csv/csv_record_file_writer_spec.rb

@@ -0,0 +1,100 @@
+require 'spec_helper'
+
+describe DataForge::File::CSV::CSVRecordFileWriter do
+
+  let(:csv_file) { instance_double "CSV" }
+  let(:definition) { instance_double "DataForge::File::CSV::CSVRecordFileDefinition",
+                                     name: :test,
+                                     file_name: "test.csv",
+                                     field_names: [:field1, :field2, :field3],
+                                     delimiter: "delimiter",
+                                     quote: "quote",
+                                     encoding: "encoding",
+                                     has_header_row: true }
+
+  subject { described_class.new definition }
+
+  before do
+    allow(Dir::Tmpname).to receive(:make_tmpname).with(["test", ".csv"], 1).and_return("generated_tempname.csv")
+    allow(CSV).to receive(:open).and_return csv_file
+  end
+
+
+  describe "#definition" do
+    it "should return the file definition the writer was created for" do
+      expect(subject.definition).to eq definition
+    end
+  end
+
+
+  describe "#name" do
+    it "should return the file definition's name" do
+      expect(subject.name).to eq :test
+    end
+  end
+
+
+  describe "#fields" do
+    it "should return the file definition's fields" do
+      expect(subject.fields).to eq [:field1, :field2, :field3]
+    end
+  end
+
+
+  describe "#open" do
+    it "should open a CSV file for writing with a temporary filename" do
+      expect(CSV).to receive(:open).with("generated_tempname.csv", "w", anything)
+
+      subject.open
+    end
+
+
+    it "should use the file definition's settings as CSV options" do
+      expect(CSV).to receive(:open).with(anything, "w", {col_sep: "delimiter",
+                                                         quote_char: "quote",
+                                                         encoding: "encoding",
+                                                         write_headers: true,
+                                                         headers: [:field1, :field2, :field3]})
+
+      subject.open
+    end
+
+
+    context "when a file has no header" do
+      it "should open a CSV file with no header row" do
+        allow(definition).to receive(:has_header_row).and_return false
+
+        expect(CSV).to receive(:open).with(anything, "w", {col_sep: "delimiter",
+                                                           quote_char: "quote",
+                                                           encoding: "encoding",
+                                                           write_headers: false})
+
+        subject.open
+      end
+    end
+  end
+
+
+  describe "#close" do
+    it "should close and rename the open file" do
+      subject.open
+
+      expect(csv_file).to receive :close
+      expect(FileUtils).to receive(:move).with("generated_tempname.csv", "test.csv")
+
+      subject.close
+    end
+  end
+
+
+  describe "#output_record" do
+    it "should write the specified fields of a hash (in the specified order) as a row into the CSV file" do
+      subject.open
+
+      expect(csv_file).to receive(:<<).with ["a", "b", "c"]
+
+      subject.write(field3: "c", field1: "a", field4: "d", field2: "b")
+    end
+  end
+
+end

data/spec/data_forge/file/record_file_definition_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+describe DataForge::File::RecordFileDefinition do
+
+  describe ".from_input" do
+    it "should instantiate a CSV file definition and initalize it with the initializer block" do
+      definition = instance_double "DataForge::File::CSV::CSVRecordFileDefinition"
+      initializer_block = lambda {}
+
+      expect(DataForge::File::CSV::CSVRecordFileDefinition).to receive(:new).with(:test).and_return definition
+      expect(definition).to receive(:instance_eval) { |&block| expect(block).to be initializer_block }
+
+      expect(subject.from_input :test, &initializer_block).to eq definition
+    end
+  end
+
+end

data/spec/data_forge/file/record_file_reader_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe DataForge::File::RecordFileReader do
+
+  describe ".for" do
+    it "should return a record reader for the specified file definition" do
+      definition = instance_double DataForge::File::CSV::CSVRecordFileDefinition
+
+      expect(DataForge::File::CSV::CSVRecordFileReader).to receive(:new).with(definition).and_return "record reader"
+
+      expect(described_class.for definition).to eq "record reader"
+    end
+  end
+
+end

data/spec/data_forge/file/record_file_writer_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe DataForge::File::RecordFileWriter do
+
+  describe ".for" do
+    it "should return a record writer for the specified file definition" do
+      definition = instance_double DataForge::File::CSV::CSVRecordFileDefinition
+
+      expect(DataForge::File::CSV::CSVRecordFileWriter).to receive(:new).with(definition).and_return "record writer"
+
+      expect(described_class.for definition).to eq "record writer"
+    end
+  end
+
+end

data/spec/data_forge/file_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper'
+
+describe DataForge::File do
+
+  let(:definition) { instance_double "DataForge::File::RecordFileDefinition" }
+
+  before do
+    allow(DataForge::File::RecordFileDefinition).to receive(:from_input).with(:definition_name).and_return definition
+  end
+
+  after do
+    subject.instance_variable_set :@file_definitions, {}
+  end
+
+
+  describe ".reader_for" do
+    let(:reader) { double "Reader" }
+
+    it "should return a record reader for the file with the specified name" do
+      subject.register_file_definition :definition_name
+
+      expect(DataForge::File::RecordFileReader).to receive(:for).with(definition).and_return reader
+
+      expect(subject.reader_for :definition_name).to eq reader
+    end
+
+    it "should raise an error if there is no file registered by the specified name" do
+      expect { subject.reader_for :definition_name }.to raise_error "Unknown file reference 'definition_name'"
+    end
+  end
+
+
+  describe ".writer_for" do
+    let(:writer) { double "Writer" }
+
+    it "should return a record writer for the file with the specified name" do
+      subject.register_file_definition :definition_name
+
+      expect(DataForge::File::RecordFileWriter).to receive(:for).with(definition).and_return writer
+
+      expect(subject.writer_for :definition_name).to eq writer
+    end
+
+    it "should raise an error if there is no file registered by the specified name" do
+      expect { subject.writer_for :definition_name }.to raise_error "Unknown file reference 'definition_name'"
+    end
+  end
+
+end