masamune 0.14.0 → 0.15.0

data/spec/masamune/template_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Template do
   describe '.render_to_file' do
     let(:parameters) { {} }

data/spec/masamune/thor_spec.rb

@@ -20,26 +20,37 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-require 'thor'
-
 describe Masamune::Thor do
   let(:thor_class) do
     Class.new(Thor) do
       include Masamune::Thor
       include Masamune::Actions::DataFlow
 
+      namespace :example
+
       desc 'command', 'command'
-      target path: "target/%Y-%m-%d"
-      source path: "source/%Y%m%d*.log"
-      method_option :zombo, desc: 'Anything is possible'
-      def command
+      target path: fs.path(:tmp_dir, "target/%Y-%m-%d")
+      source path: fs.path(:tmp_dir, "source/%Y%m%d*.log")
+      def command_task
         # NOP
       end
 
       desc 'other', 'other'
       skip
-      def other
+      def other_task
+        # NOP
+      end
+
+      desc 'current_dir', 'current_dir'
+      skip
+      def current_dir_task
+        console(fs.path(:current_dir))
+      end
+
+      desc 'unknown', 'unknown'
+      target path: fs.path(:unknown_dir, "target/%Y-%m-%d")
+      source path: fs.path(:unknown_dir, "source/%Y%m%d*.log")
+      def unknown_task
         # NOP
       end
     end
@@ -188,6 +199,40 @@ describe Masamune::Thor do
       end
       it { expect { cli_invocation }.to raise_error /random exception/ }
     end
+
+    context 'with command that raises exception during execution' do
+      let(:command) { 'unknown' }
+      let(:options) { ['--start', '2013-01-01'] }
+      it 'exits with status code 1 and prints error to stderr' do
+        expect { cli_invocation }.to raise_error { |e|
+          expect(e).to be_a(SystemExit)
+          expect(e.status).to eq(1)
+        }
+        expect(stdout.string).to be_blank
+        expect(stderr.string).to match(/Path :unknown_dir not defined/)
+      end
+    end
+
+  end
+
+  context 'with command that prints :current_dir' do
+    let(:thor_class) do
+      Class.new(Thor) do
+        include Masamune::Thor
+
+        desc 'current_dir', 'current_dir'
+        def current_dir
+          console(fs.path(:current_dir))
+        end
+      end
+    end
+
+    let(:command) { 'current_dir' }
+    it 'prints :current_dir' do
+      cli_invocation
+      expect(stdout.string).to eq(File.dirname(__FILE__) + "\n")
+      expect(stderr.string).to be_blank
+    end
   end
 
   context '.parse_extra' do

data/spec/masamune/transform/bulk_upsert.dimension_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::BulkUpsert do
   before do
     allow_any_instance_of(Masamune::Schema::Table).to receive(:lock_id).and_return(42)

data/spec/masamune/transform/consolidate_dimension_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::ConsolidateDimension do
   before do
     catalog.schema :postgres do

data/spec/masamune/transform/deduplicate_dimension_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::DeduplicateDimension do
   before do
     catalog.schema :postgres do

data/spec/masamune/transform/define_schema_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::DefineSchema do
   context 'for postgres schema' do
     before do

data/spec/masamune/transform/define_table.dimension_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::DefineTable do
   subject { transform.define_table(table).to_s }
 

data/spec/masamune/transform/define_table.fact_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::DefineTable do
   before do
     catalog.schema :postgres do

data/spec/masamune/transform/define_table.table_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::DefineTable do
   let(:files) { [] }
   let(:section) { :all }

data/spec/masamune/transform/denormalize_table_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::DenormalizeTable do
   before do
     catalog.schema :postgres do

data/spec/masamune/transform/insert_reference_values.dimension_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::InsertReferenceValues do
   before do
     allow_any_instance_of(Masamune::Schema::Table).to receive(:lock_id).and_return(42)

data/spec/masamune/transform/insert_reference_values.fact_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::InsertReferenceValues do
   before do
     allow_any_instance_of(Masamune::Schema::Table).to receive(:lock_id).and_return(42)

data/spec/masamune/transform/load_dimension_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::LoadDimension do
   before do
     catalog.schema :postgres do

data/spec/masamune/transform/load_fact_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::LoadFact do
   before do
     catalog.schema :postgres do

data/spec/masamune/transform/relabel_dimension_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::RelabelDimension do
   before do
     allow_any_instance_of(Masamune::Schema::Table).to receive(:lock_id).and_return(42)

data/spec/masamune/transform/rollup_fact_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::RollupFact do
   before do
     allow_any_instance_of(Masamune::Schema::Table).to receive(:lock_id).and_return(42)

data/spec/masamune/transform/snapshot_dimension_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::SnapshotDimension do
   before do
     catalog.schema :postgres do

data/spec/masamune/transform/stage_dimension_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::StageDimension do
   before do
     catalog.schema :postgres do

data/spec/masamune/transform/stage_fact_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Transform::StageFact do
   before do
     allow_any_instance_of(Masamune::Schema::Table).to receive(:lock_id).and_return(42)

data/spec/masamune_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune do
   it { is_expected.to be_a(Module) }
 

data/spec/spec_helper.rb

@@ -30,6 +30,8 @@ Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
 
 require 'active_support/core_ext/string/strip'
 
+ENV['MASAMUNE_ENV'] = 'test'
+
 Masamune::ExampleGroup.configure do |config|
   config.quiet    = ENV['MASAMUNE_DEBUG'] ? false : true
   config.debug    = ENV['MASAMUNE_DEBUG'] ? true : false

data/spec/support/masamune/job_example_group.rb

@@ -0,0 +1,62 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::JobExampleGroup
+  module JobFixtureContext
+    shared_context 'job_fixture' do |context_options = {}|
+      fixture_file = example_fixture_file(context_options.slice(:fixture, :file, :path))
+      let(:fixture) { example_fixture(file: fixture_file) }
+
+      before :all do
+        load_example_config!
+        clean_example_run!
+      end
+
+      before do
+        setup_example_input!(fixture)
+      end
+
+      it "should match #{fixture_file}" do
+        aggregate_failures 'generates expected output' do
+          gather_example_output(fixture) do |actual_data, expect_file, expect_data|
+            expect(File.exists?(expect_file)).to eq(true)
+            expect(actual_data).to eq(expect_data)
+          end
+        end
+      end
+    end
+  end
+
+  def self.included(base)
+    base.send(:include, Masamune::ExampleGroup)
+    base.send(:include, Masamune::SharedExampleGroup)
+    base.send(:include, Masamune::Actions::Filesystem)
+    base.send(:include, Masamune::Actions::Hive)
+    base.send(:include, Masamune::Actions::Postgres)
+    base.send(:include, JobFixtureContext)
+  end
+end
+
+RSpec.configure do |config|
+  config.include Masamune::JobExampleGroup, :type => :job, :file_path => %r{.*/spec/.*job_spec\.rb}
+  config.include Masamune::JobExampleGroup, :type => :task, :file_path => %r{.*/spec/.*task_spec\.rb}
+end

data/spec/support/masamune/job_fixture.rb

@@ -0,0 +1,137 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'active_support/core_ext/string/indent'
+
+module Masamune
+  class JobFixture
+    INDENT = 2
+
+    attr_accessor :name
+
+    def initialize(options = {})
+      @path     = options[:path]
+      @name     = options[:name] || options[:fixture]
+      @file     = options[:file]
+      @data     = options[:data]
+      @type     = options[:type]
+      @context  = options[:context]
+
+      @data['inputs']  ||= []
+      @data['outputs'] ||= []
+    end
+
+    class << self
+      def load(options = {}, context = binding)
+        file = file_name(options)
+        raise ArgumentError, "Fixture '#{file}' does not exist" unless File.exists?(file)
+        YAML.load(ERB.new(File.read(file)).result(context)).tap do |data|
+          return new(options.merge(data: data, context: context))
+        end
+      end
+
+      def file_name(options = {})
+        return options[:file] if options[:file]
+        File.join(options[:path], [options[:name] || options[:fixture], suffix(options)].compact.join('.'))
+      end
+
+      protected
+
+      def suffix(options = {})
+        "#{options[:type] || 'job'}_fixture.yml"
+      end
+    end
+
+    def file_name
+      self.class.file_name(file: @file, path: @path, name: @name, type: @type)
+    end
+
+    def path
+      @path || File.dirname(@file)
+    end
+
+    def save
+      FileUtils.mkdir_p(path)
+      File.open(file_name, 'w') do |file|
+        file.puts '---'
+        file.puts 'inputs:'
+        @data['inputs'].each do |input|
+          file.puts '-'.indent(INDENT)
+          serialize(input) do |elem|
+            file.puts elem.indent(INDENT*2)
+          end
+        end
+        file.puts
+        file.puts 'outputs:'
+        @data['outputs'].each do |output|
+          file.puts '-'.indent(INDENT)
+          serialize(output) do |elem|
+            file.puts elem.indent(INDENT*2)
+          end
+        end
+      end
+      file_name
+    end
+
+    def inputs
+      @inputs ||= begin
+        @data['inputs'].map do |input|
+          if input['reference']
+            raise ArgumentError, "reference in #{file_name} requires fixture" unless input['reference']['fixture'] || input['reference']['file']
+            reference = self.class.load({path: input['reference']['path'] || path, name: input['reference']['fixture'], file: input['reference']['file'], type: @type}, @context)
+            section = input['reference']['section'] || 'outputs'
+            reference.send(section) if reference.respond_to?(section)
+          else
+            input
+          end
+        end.flatten.compact
+      end
+    end
+
+    def outputs
+      @data['outputs']
+    end
+
+    private
+
+    def serialize(hash, level = 0)
+      hash.each do |key, value|
+        case value
+        when String
+          if value.split("\n").count > 1
+            yield "#{key}: |".indent(level * INDENT)
+            value.split("\n").each do |line|
+              yield line.strip.indent((level + 1) * INDENT)
+            end
+          else
+            yield "#{key}: #{value}".indent(level * INDENT)
+          end
+        when Hash
+          yield "#{key}:".indent(level * INDENT)
+          serialize(value, level + 1) do |next_hash|
+            yield next_hash
+          end
+        end
+      end
+    end
+  end
+end