masamune 0.14.0 → 0.15.0

data/spec/masamune/rspec/job_fixture_spec.rb

@@ -0,0 +1,365 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+describe Masamune::JobFixture do
+  let!(:tmp_dir) { File.join(Dir.tmpdir, SecureRandom.hex) }
+  let(:fixture_path) { File.join(tmp_dir, 'example', 'spec') }
+
+  describe '.file_name' do
+    subject { described_class.file_name(options) }
+    context 'with file' do
+      let(:options) { { file: File.join(fixture_path, 'job_fixture.yml') } }
+      it { is_expected.to eq(File.join(fixture_path, 'job_fixture.yml')) }
+    end
+
+    context 'with path' do
+      let(:options) { { path: fixture_path } }
+      it { is_expected.to eq(File.join(fixture_path, 'job_fixture.yml')) }
+    end
+
+    context 'with path and type' do
+      let(:options) { { path: fixture_path, type: 'task' } }
+      it { is_expected.to eq(File.join(fixture_path, 'task_fixture.yml')) }
+    end
+
+    context 'with path and name' do
+      let(:options) { { path: fixture_path, name: 'basic' } }
+      it { is_expected.to eq(File.join(fixture_path, 'basic.job_fixture.yml')) }
+    end
+  end
+
+  describe '#save' do
+    let(:instance) { described_class.new(path: fixture_path, data: data) }
+    let(:data) do
+      {
+        'inputs' => [
+          {
+            'file' => 'input_file',
+            'data' => 'input_data'
+          },
+          {
+            'file' => 'another_input_file',
+            'data' => <<-EOS.strip_heredoc
+              more_data
+              more_data
+            EOS
+          },
+          {
+            'reference' => {
+              'fixture' => 'other',
+              'section' => 'output'
+            }
+          }
+        ],
+        'outputs' => [
+          {
+            'file' => 'output_file',
+            'data' => 'output_data'
+          },
+          {
+            'file' => 'another_output_file',
+            'data' => <<-EOS.strip_heredoc
+              more_data
+              more_data
+            EOS
+          }
+        ]
+      }
+    end
+
+    before do
+      instance.save
+    end
+
+    subject { File.read(instance.file_name) }
+
+    it 'saves pretty fixture' do
+      is_expected.to eq <<-EOS.strip_heredoc
+        ---
+        inputs:
+          -
+            file: input_file
+            data: input_data
+          -
+            file: another_input_file
+            data: |
+              more_data
+              more_data
+          -
+            reference:
+              fixture: other
+              section: output
+
+        outputs:
+          -
+            file: output_file
+            data: output_data
+          -
+            file: another_output_file
+            data: |
+              more_data
+              more_data
+      EOS
+    end
+  end
+
+  describe '.load' do
+    let(:basic_data) do
+      {
+        'inputs' => [
+          {
+            'file' => 'basic_input_file',
+            'data' => 'basic_input_data'
+          }
+        ],
+        'outputs' => [
+          {
+            'file' => 'basic_output_file',
+            'data' => 'basic_output_data'
+          }
+        ]
+      }
+    end
+
+    let(:other_data) { {} }
+
+    before do
+      described_class.new(path: fixture_path, name: 'basic', data: basic_data).save
+      described_class.new(path: fixture_path, name: 'other', data: other_data).save
+    end
+
+    context 'with basic fixture from path' do
+      subject(:instance) { described_class.load(path: fixture_path, name: 'basic') }
+
+      it 'loads basic fixture' do
+        expect(instance.inputs).to eq(basic_data['inputs'])
+        expect(instance.outputs).to eq(basic_data['outputs'])
+      end
+    end
+
+    context 'with basic fixture from file' do
+      subject(:instance) { described_class.load(file: File.join(fixture_path, 'basic.job_fixture.yml')) }
+
+      it 'loads basic fixture' do
+        expect(instance.inputs).to eq(basic_data['inputs'])
+        expect(instance.outputs).to eq(basic_data['outputs'])
+      end
+    end
+
+    context 'with basic fixture from path that does not exist' do
+      subject(:instance) { described_class.load(path: fixture_path, name: 'unknown') }
+
+      it { expect { instance }.to raise_error(ArgumentError) }
+    end
+
+    context 'with basic fixture from file that does not exist' do
+      subject(:instance) { described_class.load(file: File.join(fixture_path, 'unknown.job_fixture.yml')) }
+
+      it { expect { instance }.to raise_error(ArgumentError) }
+    end
+
+    context 'with reference fixture' do
+      let(:other_data) do
+        {
+          'inputs' => [
+            {
+              'file' => 'other_input_file',
+              'data' => 'other_input_data'
+            },
+            {
+              'reference' => {
+                'fixture' => 'basic'
+              }
+            }
+          ],
+          'outputs' => [
+            {
+              'file' => 'other_output_file',
+              'data' => 'other_output_data'
+            }
+          ]
+        }
+      end
+
+      subject(:instance) { described_class.load(path: fixture_path, name: 'other') }
+
+      it 'loads referenced fixture' do
+        expect(instance.inputs).to include(other_data['inputs'].first)
+        expect(instance.inputs).to include(basic_data['outputs'].first)
+        expect(instance.outputs).to eq(other_data['outputs'])
+      end
+    end
+
+    context 'with reference fixture and path' do
+      let(:other_data) do
+        {
+          'inputs' => [
+            {
+              'file' => 'other_input_file',
+              'data' => 'other_input_data'
+            },
+            {
+              'reference' => {
+                'path' => fixture_path,
+                'fixture' => 'basic'
+              }
+            }
+          ],
+          'outputs' => [
+            {
+              'file' => 'other_output_file',
+              'data' => 'other_output_data'
+            }
+          ]
+        }
+      end
+
+      subject(:instance) { described_class.load(path: fixture_path, name: 'other') }
+
+      it 'loads referenced fixture' do
+        expect(instance.inputs).to include(other_data['inputs'].first)
+        expect(instance.inputs).to include(basic_data['outputs'].first)
+        expect(instance.outputs).to eq(other_data['outputs'])
+      end
+    end
+
+    context 'with reference fixture from file' do
+      let(:other_data) do
+        {
+          'inputs' => [
+            {
+              'file' => 'other_input_file',
+              'data' => 'other_input_data'
+            },
+            {
+              'reference' => {
+                'file' => File.join(fixture_path, 'basic.job_fixture.yml')
+              }
+            }
+          ],
+          'outputs' => [
+            {
+              'file' => 'other_output_file',
+              'data' => 'other_output_data'
+            }
+          ]
+        }
+      end
+
+      subject(:instance) { described_class.load(path: fixture_path, name: 'other') }
+
+      it 'loads referenced fixture' do
+        expect(instance.inputs).to include(other_data['inputs'].first)
+        expect(instance.inputs).to include(basic_data['outputs'].first)
+        expect(instance.outputs).to eq(other_data['outputs'])
+      end
+    end
+
+
+    context 'with reference fixture and section' do
+      let(:other_data) do
+        {
+          'inputs' => [
+            {
+              'file' => 'other_input_file',
+              'data' => 'other_input_data'
+            },
+            {
+              'reference' => {
+                'fixture' => 'basic',
+                'section' => 'inputs'
+              }
+            }
+          ],
+          'outputs' => [
+            {
+              'file' => 'other_output_file',
+              'data' => 'other_output_data'
+            }
+          ]
+        }
+      end
+
+      subject(:instance) { described_class.load(path: fixture_path, name: 'other') }
+
+      it 'loads referenced fixture' do
+        expect(instance.inputs).to include(other_data['inputs'].first)
+        expect(instance.inputs).to include(basic_data['inputs'].first)
+        expect(instance.outputs).to eq(other_data['outputs'])
+      end
+    end
+
+    context 'with reference fixture that does not exist' do
+      let(:other_data) do
+        {
+          'inputs' => [
+            {
+              'file' => 'other_input_file',
+              'data' => 'other_input_data'
+            },
+            {
+              'reference' => {
+                'fixture' => 'unknown'
+              }
+            }
+          ],
+          'outputs' => [
+            {
+              'file' => 'other_output_file',
+              'data' => 'other_output_data'
+            }
+          ]
+        }
+      end
+
+      subject(:instance) { described_class.load(path: fixture_path, name: 'other') }
+
+      it { expect { instance.inputs }.to raise_error(ArgumentError) }
+    end
+
+    context 'with invalid reference fixture' do
+      let(:other_data) do
+        {
+          'inputs' => [
+            {
+              'file' => 'other_input_file',
+              'data' => 'other_input_data'
+            },
+            {
+              'reference' => 'unknown'
+            }
+          ],
+          'outputs' => [
+            {
+              'file' => 'other_output_file',
+              'data' => 'other_output_data'
+            }
+          ]
+        }
+      end
+
+      subject(:instance) { described_class.load(path: fixture_path, name: 'other') }
+
+      it { expect { instance.inputs }.to raise_error(ArgumentError) }
+    end
+  end
+end

data/spec/masamune/rspec/shared_example_group_spec.rb

@@ -0,0 +1,73 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+describe Masamune::SharedExampleGroup do
+  it { is_expected.to be_a(Module) }
+
+  let(:klass) { Class.new.send(:include, described_class) }
+
+  describe '.example_fixture_file' do
+    before do
+      allow(klass).to receive(:file_path).and_return(file_path)
+    end
+
+    subject { klass.example_fixture_file(options) }
+
+    context 'with file_path like task_spec.rb' do
+      let(:file_path) { './examples/apache_log/spec/task_spec.rb' }
+
+      context 'with options empty' do
+        let(:options) { {} }
+        it { is_expected.to eq('./examples/apache_log/spec/task_fixture.yml') }
+      end
+
+      context 'with options fixture:' do
+        let(:options) { { fixture: 'fixture_name' } }
+        it { is_expected.to eq('./examples/apache_log/spec/fixture_name.task_fixture.yml') }
+      end
+
+      context 'with options file:' do
+        let(:options) { { file: './examples/apache_log/spec/task_fixture.yml' } }
+        it { is_expected.to eq('./examples/apache_log/spec/task_fixture.yml') }
+      end
+    end
+
+    context 'with file_path like mapper_spec.rb' do
+      let(:file_path) { './examples/apache_log/spec/mapper_spec.rb' }
+
+      context 'with options empty' do
+        let(:options) { {} }
+        it { is_expected.to eq('./examples/apache_log/spec/mapper_fixture.yml') }
+      end
+
+      context 'with options fixture:' do
+        let(:options) { { fixture: 'fixture_name' } }
+        it { is_expected.to eq('./examples/apache_log/spec/fixture_name.mapper_fixture.yml') }
+      end
+
+      context 'with options file:' do
+        let(:options) { { file: './examples/apache_log/spec/mapper_fixture.yml' } }
+        it { is_expected.to eq('./examples/apache_log/spec/mapper_fixture.yml') }
+      end
+    end
+  end
+end

data/spec/masamune/schema/catalog_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Schema::Catalog do
   let(:environment) { double }
   let(:instance) { described_class.new(environment) }
@@ -145,6 +143,20 @@ describe Masamune::Schema::Catalog do
       it { expect(table_two_columns).to include :column_four }
     end
 
+    context 'when schema contains columns with symbol names' do
+      before do
+        instance.schema :postgres do
+          dimension :table_one, type: :two do
+            column :column_one
+          end
+        end
+      end
+
+      let(:table_one_columns) { postgres.table_one_dimension.columns }
+
+      it { expect(table_one_columns).to include :column_one }
+    end
+
     context 'when schema contains columns and rows' do
       before do
         instance.schema :postgres do

data/spec/masamune/schema/column_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Schema::Column do
   describe '.initialize' do
     subject(:column) { described_class.new(id: 'id') }

data/spec/masamune/schema/dimension_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Schema::Dimension do
   let(:store) { double }
 

data/spec/masamune/schema/fact_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Schema::Fact do
   let(:store) { double(id: 'store', type: :postgres) }
 

data/spec/masamune/schema/map_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Schema::Map do
   let(:environment) { double(logger: double) }
   let(:catalog) { Masamune::Schema::Catalog.new(environment) }
@@ -547,6 +545,57 @@ describe Masamune::Schema::Map do
 
       it_behaves_like 'apply input/output'
     end
+
+    context 'with nested json and indifferent access' do
+      before do
+        catalog.schema :files do
+          file 'input' do
+            column 'data', type: :json
+          end
+
+          file 'output' do
+            column 'id', type: :integer
+          end
+
+          map from: files.input, to: files.output do |row|
+            [
+              { 'id' => row[:data][:user][:id] },
+              { 'id' => row['data']['user']['id'] }
+            ]
+          end
+        end
+      end
+
+      let(:source) do
+        catalog.files.input
+      end
+
+      let(:target) do
+        catalog.files.output
+      end
+
+      let(:source_data) do
+        <<-EOS.strip_heredoc
+          {"user":{"id":1}}
+          {"user":{"id":2}}
+        EOS
+      end
+
+      let(:target_data) do
+        <<-EOS.strip_heredoc
+          1
+          1
+          2
+          2
+        EOS
+      end
+
+      it 'should match target data' do
+        is_expected.to eq(target_data)
+      end
+
+      it_behaves_like 'apply input/output'
+    end
   end
 
   describe Masamune::Schema::Map::JSONEncoder do

data/spec/masamune/schema/row_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Schema::Row do
   subject(:row) { described_class.new }
   it { expect(row).to_not be_nil }

data/spec/masamune/schema/store_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Schema::Store do
   let(:environment) { double }
 
@@ -39,11 +37,34 @@ describe Masamune::Schema::Store do
     subject(:store) { described_class.new(environment, type: :postgres) }
     it { expect(store.format).to eq(:csv) }
     it { expect(store.headers).to be_truthy }
+    it { expect(store.json_encoding).to eq(:quoted) }
+
+    context 'with format override' do
+      subject(:store) { described_class.new(environment, type: :postgres, format: :raw) }
+      it { expect(store.format).to eq(:raw) }
+      it { expect(store.headers).to be_falsey }
+      it { expect(store.json_encoding).to eq(:raw) }
+    end
   end
 
   context 'with type :hive' do
     subject(:store) { described_class.new(environment, type: :hive) }
     it { expect(store.format).to eq(:tsv) }
     it { expect(store.headers).to be_falsey }
+    it { expect(store.json_encoding).to eq(:raw) }
+  end
+
+  context 'with type :files' do
+    subject(:store) { described_class.new(environment, type: :files) }
+    it { expect(store.format).to eq(:raw) }
+    it { expect(store.headers).to be_falsey }
+    it { expect(store.json_encoding).to eq(:raw) }
+
+    context 'with format overrides' do
+      subject(:store) { described_class.new(environment, type: :files, format: :csv, headers: true, json_encoding: :quoted) }
+      it { expect(store.format).to eq(:csv) }
+      it { expect(store.headers).to be_truthy }
+      it { expect(store.json_encoding).to eq(:quoted) }
+    end
   end
 end

data/spec/masamune/schema/table_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::Schema::Table do
   context 'without id' do
     subject(:table) { described_class.new }

data/spec/masamune/string_format_spec.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-
 describe Masamune::StringFormat do
   let(:instance) { Object.new.extend(described_class) }
 

data/spec/masamune/tasks/dump_thor_spec.rb

@@ -20,9 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-require 'thor'
-
 require 'masamune/tasks/dump_thor'
 
 describe Masamune::Tasks::DumpThor do

data/spec/masamune/tasks/elastic_mapreduce_thor_spec.rb

@@ -20,9 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-require 'thor'
-
 require 'masamune/tasks/elastic_mapreduce_thor'
 
 describe Masamune::Tasks::ElasticMapreduceThor do

data/spec/masamune/tasks/hive_thor_spec.rb

@@ -20,9 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-require 'thor'
-
 require 'masamune/tasks/hive_thor'
 
 describe Masamune::Tasks::HiveThor do

data/spec/masamune/tasks/postgres_thor_spec.rb

@@ -20,9 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-require 'thor'
-
 require 'masamune/tasks/postgres_thor'
 
 describe Masamune::Tasks::PostgresThor do

data/spec/masamune/tasks/shell_thor_spec.rb

@@ -20,9 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'spec_helper'
-require 'thor'
-
 require 'masamune/tasks/shell_thor'
 
 describe Masamune::Tasks::ShellThor do