masamune 0.14.0 → 0.15.0

data/spec/support/masamune/shared_example_group.rb

@@ -0,0 +1,203 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::SharedExampleGroup
+  extend ActiveSupport::Concern
+
+  def capture_popen(cmd, stdin)
+    stdout = StringIO.new
+    IO.popen(cmd, 'r+') do |io|
+      io.sync = true
+      io.write stdin
+      io.close_write
+      while !io.eof?
+        stdout << io.read
+      end
+    end
+    stdout.string
+  end
+
+  def capture_output
+    @stdout, @stderr = StringIO.new, StringIO.new
+    tmp_stdout, $stdout = $stdout, @stdout
+    tmp_stderr, $stderr = $stderr, @stderr
+    yield
+  rescue SystemExit => e
+    @status = e.status
+  ensure
+    $stdout, $stderr = tmp_stdout, tmp_stderr
+  end
+
+  def capture(enable = true)
+    if enable
+      capture_output do
+        yield
+      end
+    else
+      yield
+    end
+  end
+
+  def with_delim(data, delim = nil)
+    return data unless delim
+    case delim
+    when Hash
+      delim.each do |from, to|
+        data.gsub!(from, to)
+      end
+    when Array
+      data.gsub!(*delim)
+    end
+    data
+  end
+
+  def load_example_config!
+    filesystem.add_path(:current_dir, self.class.example_current_dir) if self.class.example_current_dir
+    environment.configuration.load(self.class.example_default_config) if self.class.example_default_config
+  end
+
+  # TODO: iterate over databases
+  def clean_example_run!
+    if configuration.postgres[:clean]
+      postgres_admin(action: :drop, database: configuration.postgres[:database])
+      postgres_admin(action: :create, database: configuration.postgres[:database])
+      postgres(file: define_schema(catalog, :postgres).to_file, retries: 0)
+    end
+    filesystem.paths.each do |_, (path, options)|
+      filesystem.remove_dir(path) if options[:clean]
+    end
+  end
+
+  # TODO encapsulate commands as runners
+  def setup_example_input!(fixture)
+    fixture.inputs.each do |input|
+      if input['file']
+        filesystem.write(with_delim(input['data'], input['delim']), input['file'])
+      end
+
+      if input['hive']
+        hive(exec: input['hive'])
+      end
+
+      if input['psql']
+        Tempfile.open('etl') do |tmp|
+          tmp.write(input['psql'])
+          tmp.flush
+          postgres(file: tmp.path)
+        end
+      end
+    end
+  end
+
+  def gather_example_output(fixture)
+    fail "No outputs defined for #{fixture.file_name}" if fixture.outputs.none?
+    fixture.outputs.each do |output|
+      output_file = output['file'] || Tempfile.new('masamune').path
+
+      execute_output_command(output, output_file)
+      next unless output['data']
+      actual_data = File.read(output_file).strip
+      expect_data = with_delim(output['data'].strip, output['delim']).strip
+      if output['order'] == 'random'
+        actual_data = actual_data.split("\n").sort.join("\n")
+        expect_data = expect_data.split("\n").sort.join("\n")
+      end
+      yield [actual_data, output_file, expect_data]
+      FileUtils.rm(output_file) if File.exists?(output_file)
+    end
+  end
+
+  def execute_output_command(output, output_file)
+    if output['hive'] && output['hive'].is_a?(String)
+      # FIXME: Replace with exec once SBI-530 is fixed
+      Tempfile.open('etl') do |tmp|
+        tmp.write(output['hive'])
+        tmp.flush
+        hive(file: tmp.path, output: output_file)
+      end
+    elsif output['table']
+      table = eval "catalog.#{output['table']}"
+      query = denormalize_table(table, output.slice('columns', 'order', 'except', 'include')).to_s
+      # FIXME: define format based on table.store.format
+      case table.store.type
+      when :postgres
+        postgres(exec: query, csv: true, output: output_file)
+      when :hive
+        # FIXME: Replace with exec once SBI-530 is fixed
+        Tempfile.open('etl') do |tmp|
+          tmp.write(query)
+          tmp.flush
+          hive(file: tmp.path, output: output_file)
+        end
+      else
+        raise "'table' output not supported for #{output['table']}"
+      end
+    elsif output['hive'] && output['hive'].is_a?(Hash)
+      hive(file: example_file(output['hive']['file']), output: output_file)
+    elsif output['psql'] && output['psql'].is_a?(String)
+      postgres(exec: output['psql'], csv: true, output: output_file)
+    elsif output['psql'] && output['psql'].is_a?(Hash)
+      postgres(file: example_file(output['psql']['file']), variables: output['psql'].fetch('variables', {}), csv: true, output: output_file)
+    end
+  end
+
+  def example_fixture(options = {})
+    Masamune::JobFixture.load(options.merge(type: self.class.example_type), binding)
+  end
+
+  module ClassMethods
+    def example_current_dir
+      example_file_path_info[:current_dir]
+    end
+
+    def example_default_config
+      return unless respond_to?(:described_class)
+      return unless described_class.respond_to?(:class_options)
+      described_class.class_options[:config].try(:default)
+    end
+
+    def example_fixture_file(options = {})
+      Masamune::JobFixture.file_name(options.merge(path: File.dirname(file_path), type: example_type))
+    end
+
+    def example_step
+      return unless respond_to?(:description)
+      File.join(example_current_dir, description)
+    end
+
+    def example_type
+      example_file_path_info[:example_type]
+    end
+
+    private
+
+    EXAMPLE_FILE_PATH_INFO = %r{(?<current_dir>.*?)/spec/((?<example_name>\w+)_)?(?<example_type>\w+)_spec\.rb\z}
+    def example_file_path_info
+      return {} unless respond_to?(:file_path)
+      @example_file_path_info ||= EXAMPLE_FILE_PATH_INFO.match(file_path) || {}
+    end
+
+    def example_name
+      example_file_path_info[:example_name]
+    end
+  end
+end

data/spec/support/masamune/step_example_group.rb

@@ -0,0 +1,68 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::StepExampleGroup
+  module StepFixtureContext
+    shared_context 'step_fixture' do |context_options = {}|
+      fixture_file = example_fixture_file(context_options.slice(:fixture, :file, :path))
+      step_file = example_step
+
+      args = context_options[:args]
+      subject do
+        capture_popen([step_file, args].compact.join(' '), input)
+      end
+
+      context "with #{fixture_file} fixture" do
+        let(:fixture) { Masamune::StepFixture.load({file: fixture_file}, binding) }
+
+        let(:input) { fixture.input }
+        let(:output) { fixture.output }
+
+        it 'should match output' do
+          is_expected.to eq(output)
+        end
+
+        after(:each) do |example|
+          if example.exception && ENV['MASAMUNE_RECORD']
+            shell = Thor::Shell::Basic.new
+            shell.say(example.exception)
+            if shell.yes?('Save recording?')
+              fixture.output = subject
+              fixture.save
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def self.included(base)
+    base.send(:include, Masamune::SharedExampleGroup)
+    base.send(:include, StepFixtureContext)
+  end
+end
+
+RSpec.configure do |config|
+  config.include Masamune::StepExampleGroup, :type => :step, :file_path => %r{.*/spec/.*step_spec\.rb}
+  config.include Masamune::StepExampleGroup, :type => :step, :file_path => %r{.*/spec/.*mapper_spec\.rb}
+  config.include Masamune::StepExampleGroup, :type => :step, :file_path => %r{.*/spec/.*reducer_spec\.rb}
+end

data/spec/support/masamune/step_fixture.rb

@@ -0,0 +1,91 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'active_support/core_ext/string/indent'
+
+module Masamune
+  class StepFixture
+    INDENT = 2
+
+    def initialize(options = {})
+      @file   = options[:file]
+      @data   = options[:data]
+      @type   = options[:type] || 'step'
+
+      @data['input']  ||= []
+      @data['output'] ||= []
+    end
+
+    class << self
+      def load(options = {}, context = binding)
+        file = options[:file]
+        raise ArgumentError, "Fixture '#{file}' does not exist" unless File.exists?(file)
+        YAML.load(ERB.new(File.read(file)).result(context)).tap do |data|
+          return new(options.merge(data: data))
+        end
+      end
+
+      protected
+
+      def suffix(options = {})
+        "#{options[:type] || 'step'}_fixture.yml"
+      end
+    end
+
+    def file_name
+      @file
+    end
+
+    def path
+      @path || File.dirname(@file)
+    end
+
+    def save
+      FileUtils.mkdir_p(path)
+      File.open(file_name, 'w') do |file|
+        file.puts '---'
+        file.puts 'input: |'
+        @data['input'].split("\n").each do |line|
+          file.puts line.indent(INDENT)
+        end
+        file.puts
+        file.puts 'output: |'
+        @data['output'].split("\n").each do |line|
+          file.puts line.indent(INDENT)
+        end
+      end
+      file_name
+    end
+
+    def input
+      @data['input']
+    end
+
+    def output
+      @data['output']
+    end
+
+    def output=(output)
+      @data['output'] = output
+    end
+  end
+end

data/{lib/masamune/thor_loader.rb → spec/support/masamune/task_example_group.rb}

@@ -20,19 +20,42 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-module Masamune
-  module ThorLoader
-    def self.load_default_tasks
-      Masamune::ThorLoader.load_thorfiles(File.expand_path('../../../lib/masamune/tasks', __FILE__))
-    end
+module Masamune::TaskExampleGroup
+  module TaskFixtureContent
+    shared_context 'task_fixture' do |context_options = {}|
+      include_context 'job_fixture', context_options
+      let!(:default_options) { configuration.as_options }
+
+      let(:stdout) { @stdout }
+      let(:stderr) { @stderr }
+      let(:status) { @status }
 
-    def self.load_thorfiles(dir)
-      Dir.chdir(dir) do
-        thor_files = Dir.glob('**/*.rb').delete_if { |x| not File.file?(x) }
-        thor_files.each do |f|
-          ::Thor::Util.load_thorfile(f)
+      let(:command) { nil }
+      let(:options) { [] }
+
+      subject(:execute_command) do
+        n = context_options.fetch(:idempotent, false) ? 2 : 1
+        n = 1 if ENV['MASAMUNE_FASTER_SPEC']
+        capture(!default_options.include?('--debug')) do
+          n.times do
+            Array.wrap(command).each do |cmd|
+              described_class.start([cmd, *(default_options + options)].compact)
+            end
+          end
         end
       end
     end
   end
+
+  def self.included(base)
+    base.send(:include, Masamune::ExampleGroup)
+    base.send(:include, Masamune::Actions::Filesystem)
+    base.send(:include, Masamune::Actions::Hive)
+    base.send(:include, Masamune::Transform::DenormalizeTable)
+    base.send(:include, TaskFixtureContent)
+  end
+end
+
+RSpec.configure do |config|
+  config.include Masamune::TaskExampleGroup, :type => :task, :file_path => %r{.*/spec/.*task_spec\.rb}
 end

data/spec/support/rspec/example/action_example_group.rb

@@ -24,7 +24,7 @@ module ActionExampleGroup
   def self.included(base)
     base.let(:run_dir) { Dir.mktmpdir('masamune') }
     base.before do
-      Masamune.filesystem.add_path(:run_dir, run_dir)
+      Masamune::ExampleGroup.filesystem.add_path(:run_dir, run_dir)
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: masamune
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.15.0
 platform: ruby
 authors:
 - Michael Andrews
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-22 00:00:00.000000000 Z
+date: 2015-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -150,6 +150,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: user_agent_parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -262,7 +276,6 @@ files:
 - lib/masamune/tasks/shell_thor.rb
 - lib/masamune/template.rb
 - lib/masamune/thor.rb
-- lib/masamune/thor_loader.rb
 - lib/masamune/topological_hash.rb
 - lib/masamune/transform.rb
 - lib/masamune/transform/bulk_upsert.rb
@@ -351,6 +364,8 @@ files:
 - spec/masamune/environment_spec.rb
 - spec/masamune/filesystem_spec.rb
 - spec/masamune/helpers/postgres_spec.rb
+- spec/masamune/rspec/job_fixture_spec.rb
+- spec/masamune/rspec/shared_example_group_spec.rb
 - spec/masamune/schema/catalog_spec.rb
 - spec/masamune/schema/column_spec.rb
 - spec/masamune/schema/dimension_spec.rb
@@ -387,9 +402,15 @@ files:
 - spec/masamune_spec.rb
 - spec/spec_helper.rb
 - spec/support/masamune/example_group.rb
+- spec/support/masamune/job_example_group.rb
+- spec/support/masamune/job_fixture.rb
 - spec/support/masamune/mock_command.rb
 - spec/support/masamune/mock_delegate.rb
 - spec/support/masamune/mock_filesystem.rb
+- spec/support/masamune/shared_example_group.rb
+- spec/support/masamune/step_example_group.rb
+- spec/support/masamune/step_fixture.rb
+- spec/support/masamune/task_example_group.rb
 - spec/support/masamune/thor_mute.rb
 - spec/support/rspec/example/action_example_group.rb
 - spec/support/rspec/example/task_example_group.rb
@@ -454,6 +475,8 @@ test_files:
 - spec/masamune/environment_spec.rb
 - spec/masamune/filesystem_spec.rb
 - spec/masamune/helpers/postgres_spec.rb
+- spec/masamune/rspec/job_fixture_spec.rb
+- spec/masamune/rspec/shared_example_group_spec.rb
 - spec/masamune/schema/catalog_spec.rb
 - spec/masamune/schema/column_spec.rb
 - spec/masamune/schema/dimension_spec.rb
@@ -490,9 +513,15 @@ test_files:
 - spec/masamune_spec.rb
 - spec/spec_helper.rb
 - spec/support/masamune/example_group.rb
+- spec/support/masamune/job_example_group.rb
+- spec/support/masamune/job_fixture.rb
 - spec/support/masamune/mock_command.rb
 - spec/support/masamune/mock_delegate.rb
 - spec/support/masamune/mock_filesystem.rb
+- spec/support/masamune/shared_example_group.rb
+- spec/support/masamune/step_example_group.rb
+- spec/support/masamune/step_fixture.rb
+- spec/support/masamune/task_example_group.rb
 - spec/support/masamune/thor_mute.rb
 - spec/support/rspec/example/action_example_group.rb
 - spec/support/rspec/example/task_example_group.rb