masamune 0.11.0

data/spec/masamune/commands/s3cmd_spec.rb

@@ -0,0 +1,50 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Commands::S3Cmd do
+  let(:configuration) { {:options => options} }
+  let(:options) { [] }
+  let(:attrs) { {} }
+
+  let(:delegate) { double }
+  let(:instance) { described_class.new(delegate, attrs) }
+
+  before do
+    allow(delegate).to receive_message_chain(:configuration, :s3cmd).and_return(configuration)
+  end
+
+  describe '#command_args' do
+    let(:attrs) { {extra: ['ls', 's3://fake']} }
+
+    subject { instance.command_args }
+
+    it { is_expected.to eq(['s3cmd', 'ls', 's3://fake']) }
+
+    context 'with options' do
+      let(:options) { [{'--config' => '/opt/etc/etl/s3cfg'}] }
+
+      it { is_expected.to eq(['s3cmd', '--config', '/opt/etc/etl/s3cfg', 'ls', 's3://fake']) }
+    end
+  end
+end

data/spec/masamune/commands/shell_spec.rb

@@ -0,0 +1,101 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Commands::Shell do
+  let(:input) { nil }
+  let(:options) { {fail_fast: false} }
+  let(:delegate) { Masamune::MockDelegate.new(command, input) }
+  let(:instance) { described_class.new(delegate, options) }
+
+  describe '#execute' do
+    subject do
+      instance.execute
+    end
+
+    context 'with simple command that succeeds' do
+      let(:command) { %Q{bash -c "echo 'stdout 1'; echo 'stderr 1' 1>&2; echo 'stdout 2'; echo 'stderr 2' 1>&2"} }
+
+      before do
+        subject
+      end
+
+      it { expect(delegate.status).to eq(0) }
+      it { expect(delegate.stdout).to eq(['stdout 1', 'stdout 2']) }
+      it { expect(delegate.stderr).to eq(['stderr 1', 'stderr 2']) }
+    end
+
+    context 'with simple command that fails' do
+      let(:command) { %Q{bash -c 'exit 1'} }
+
+      before do
+        subject
+      end
+
+      it { expect(delegate.status).to eq(1) }
+      it { expect(delegate.stdout).to eq([]) }
+      it { expect(delegate.stderr).to eq([]) }
+    end
+
+    context 'with fail_fast and simple command that fails' do
+      let(:command) { %Q{bash -c 'exit 1'} }
+      let(:options) { {fail_fast: true} }
+      it { expect { subject }.to raise_error RuntimeError, "fail_fast: #{command}" }
+    end
+
+    context 'when command is interrupted' do
+      let(:command) { %Q{bash -c "echo 'test'"} }
+
+      before do
+        expect(delegate).to receive(:after_execute) { raise Interrupt }
+        subject
+      end
+
+      it { expect(delegate.status).to eq(130) }
+    end
+
+    context 'with simple command with input' do
+      let(:command) { 'cat' }
+      let(:input) { "ping\npong" }
+
+      before do
+        subject
+      end
+
+      it { expect(delegate.status).to eq(0) }
+      it { expect(delegate.stdout).to eq(['ping', 'pong']) }
+      it { expect(delegate.stderr).to eq([]) }
+    end
+
+    context 'with simple command with not-ready input' do
+      let(:command) { 'cat' }
+      let(:input) { "ping\npong" }
+
+      before do
+        expect_any_instance_of(IO).to receive(:wait_writable).and_return(nil)
+      end
+
+      it { expect { subject }.to raise_error RuntimeError, /IO stdin not ready/ }
+    end
+  end
+end

data/spec/masamune/configuration_spec.rb

@@ -0,0 +1,102 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Configuration do
+  let(:environment) { Masamune::Environment.new }
+  let(:instance) { described_class.new(environment) }
+
+  describe '.default_config_file' do
+    subject { described_class.default_config_file }
+    it { is_expected.to match(%r{config/masamune\.yml\.erb\Z}) }
+  end
+
+  describe '#default_config_file' do
+    subject { instance.default_config_file }
+    it { is_expected.to match(%r{config/masamune\.yml\.erb\Z}) }
+  end
+
+  describe '#bind_template' do
+    let(:section) { nil }
+    let(:template) { nil }
+    let(:params) { nil }
+
+    subject do
+      instance.bind_template(section, template, params)
+    end
+
+    context 'with invalid template section' do
+      let(:section) { :missing_section }
+      it { expect { subject }.to raise_error(ArgumentError) }
+    end
+
+    context 'when template section is missing' do
+      let(:section) { :elastic_mapreduce }
+      it { expect { subject }.to raise_error(ArgumentError) }
+    end
+
+    context 'with valid template section' do
+      let(:section) { :elastic_mapreduce }
+      before do
+        instance.elastic_mapreduce[:templates] = {
+          list_with_state: {
+            command: '--list --key-pair %key_pair --start %state --verbose=%verbose',
+            default: {
+              state: 'RUNNING',
+              verbose: true
+            }
+          },
+          broken_template: nil
+        }
+      end
+
+      context 'when template is missing' do
+        let(:template) { :missing_template }
+        it { expect { subject }.to raise_error(ArgumentError, 'no template for missing_template') }
+      end
+
+      context 'when template is broken' do
+        let(:template) { :broken_template }
+        it { expect { subject }.to raise_error(ArgumentError, 'no template for broken_template') }
+      end
+
+      context 'when params missing but default exists' do
+        let(:template) { :list_with_state }
+        let(:params) { {key_pair: 'emr-2013'} }
+        it { is_expected.to eq(['--list', '--key-pair', 'emr-2013', '--start', 'RUNNING', '--verbose=true']) }
+      end
+
+      context 'when params missing and default missing' do
+        let(:template) { :list_with_state }
+        let(:params) { {state: 'COMPLETED'} }
+        it { expect { subject }.to raise_error(ArgumentError, 'no param for %key_pair') }
+      end
+
+      context 'with params' do
+        let(:template) { :list_with_state }
+        let(:params) { {key_pair: 'emr-2013', state: 'COMPLETED'} }
+        it { is_expected.to eq(['--list', '--key-pair', 'emr-2013', '--start', 'COMPLETED', '--verbose=true']) }
+      end
+    end
+  end
+end

data/spec/masamune/data_plan/builder_spec.rb

@@ -0,0 +1,91 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::DataPlan::Builder do
+  describe '#build' do
+    subject(:engine) do
+      described_class.instance.build(namespaces, commands, sources, targets)
+    end
+
+    context 'with multiple namespaces' do
+      let(:namespaces) { ['a', 'b'] }
+      let(:commands) { ['load', 'store'] }
+      let(:sources) { [{ path: 'log/%Y%m%d.*.log' }, { path: 'table/y=%Y/m=%m/d=%d' }] }
+      let(:targets) { [{ path: 'table/y=%Y/m=%m/d=%d' }, { path: 'daily/%Y-%m-%d' }] }
+
+      before do
+        expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:load', { path: 'table/y=%Y/m=%m/d=%d'})
+        expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:load', { path: 'log/%Y%m%d.*.log' })
+        expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('a:load', an_instance_of(Proc))
+        expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('b:store', { path: 'daily/%Y-%m-%d' })
+        expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('b:store', { path: 'table/y=%Y/m=%m/d=%d' })
+        expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('b:store', an_instance_of(Proc))
+        subject
+      end
+
+      it 'should build a Masamune::DataPlan::Engine instance' do; end
+    end
+
+    context 'with :for option' do
+      let(:namespaces) { ['a', 'a', 'a'] }
+      let(:commands) { ['missing_before', 'override', 'missing_after'] }
+      let(:sources) { [{ path: 'log/%Y%m%d.*.log', for: 'override' }] }
+      let(:targets) { [{ path: 'table/y=%Y/m=%m/d=%d', for: 'override' }] }
+
+      before do
+        expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_target_rule).with('a:missing_before', anything)
+        expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_source_rule).with('a:missing_before', anything)
+        expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_command_rule).with('a:missing_before', an_instance_of(Proc))
+        expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override', { path: 'table/y=%Y/m=%m/d=%d' })
+        expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override', { path: 'log/%Y%m%d.*.log' })
+        expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('a:override', an_instance_of(Proc))
+        expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_target_rule).with('a:missing_after', anything)
+        expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_source_rule).with('a:missing_after', anything)
+        expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_command_rule).with('a:missing_after', an_instance_of(Proc))
+        subject
+      end
+
+      it 'should build a Masamune::DataPlan::Engine instance' do; end
+    end
+
+    context 'with :skip option' do
+      let(:namespaces) { ['a', 'a', 'a'] }
+      let(:commands) { ['missing_before', 'override'] }
+      let(:sources) { [{ skip: true}, { path: 'log/%Y%m%d.*.log' }] }
+      let(:targets) { [{ skip: true}, { path: 'table/y=%Y/m=%m/d=%d' }] }
+
+      before do
+        expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_target_rule).with('a:missing_before', anything)
+        expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_source_rule).with('a:missing_before', anything)
+        expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_command_rule).with('a:missing_before', an_instance_of(Proc))
+        expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override', { path: 'table/y=%Y/m=%m/d=%d' })
+        expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override', { path: 'log/%Y%m%d.*.log' })
+        expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('a:override', an_instance_of(Proc))
+        subject
+      end
+
+      it 'should build a Masamune::DataPlan::Engine instance' do; end
+    end
+  end
+end

data/spec/masamune/data_plan/elem_spec.rb

@@ -0,0 +1,102 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::DataPlan::Elem do
+  let(:engine) { Masamune::DataPlan::Engine.new }
+  let(:name) { 'primary' }
+  let(:type) { :target }
+  let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: 'report/%Y-%m-%d/%H'}) }
+  let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: 'log/%Y%m%d.*.log'}) }
+
+  let(:start_time) { DateTime.civil(2013,07,19,11,07) }
+  let(:other_start_time) { DateTime.civil(2013,07,20,0,0) }
+
+  let(:options) { {tz: 'EST'} }
+  let(:other_options) { {tz: 'PST'} }
+
+  let(:instance) { described_class.new(rule, start_time, options) }
+
+  describe '#path' do
+    subject do
+      instance.path
+    end
+    it { is_expected.to eq('report/2013-07-19/11') }
+  end
+
+  describe '#==' do
+    subject do
+      instance == other
+    end
+
+    context 'when rule, options, and start_time match' do
+      let(:other) { described_class.new(rule, start_time, options) }
+      it { is_expected.to eq(true) }
+      it 'should have same hash' do
+        expect(instance.hash).to eq(other.hash)
+      end
+    end
+
+    context 'when rules differ' do
+      let(:other) { described_class.new(other_rule, start_time) }
+      it { is_expected.to eq(false) }
+    end
+
+    context 'when options differ' do
+      let(:other) { described_class.new(rule, start_time, other_options) }
+      it { is_expected.to eq(false) }
+    end
+
+    context 'when start_times differ' do
+      let(:other) { described_class.new(rule, other_start_time) }
+      it { is_expected.to eq(false) }
+    end
+  end
+
+  describe '#last_modified_at' do
+    let(:early) { Time.parse("2014-05-01 00:00:00 +0000") }
+    let(:later) { Time.parse("2014-06-01 00:00:00 +0000") }
+
+    subject do
+      instance.last_modified_at.utc
+    end
+
+    context 'with missing mtime' do
+      before do
+        expect(rule.engine.filesystem).to receive(:stat).with(instance.path).
+          and_return(nil)
+      end
+
+      it { is_expected.to eq(Masamune::DataPlan::Elem::MISSING_MODIFIED_AT) }
+    end
+
+    context 'with single mtime' do
+      before do
+        expect(rule.engine.filesystem).to receive(:stat).with(instance.path).
+          and_return(OpenStruct.new(mtime: early))
+      end
+
+      it { is_expected.to eq(early) }
+    end
+  end
+end