masamune 0.13.0 → 0.13.1

data/lib/masamune/transform/denormalize_table.rb

@@ -29,6 +29,8 @@ module Masamune::Transform
       columns = options[:include] || []
       columns += options[:columns] || target.denormalized_column_names
       columns -= options[:except] || []
+      columns -= ['last_modified_at']
+      columns.uniq!
       order_by = options[:order] || columns
       Operator.new(__method__, target: target, columns: columns, order_by: order_by, presenters: { postgres: Common, hive: Common })
     end
@@ -63,6 +65,7 @@ module Masamune::Transform
           column_names.each do |column_name|
             next unless column = dereference_column_name(column_name)
             next unless column.reference
+            next if column.reference.degenerate
             adjacent_reference = references[column.reference.id]
             next unless adjacent_reference
             adjacent_column = columns[adjacent_reference.foreign_key_name]

data/lib/masamune/transform/operator.rb

@@ -28,6 +28,7 @@ module Masamune::Transform
       @source     = options.delete(:source)
       @target     = options.delete(:target)
       @presenters = options.delete(:presenters) || {}
+      @helper     = options.delete(:helper)
       @locals     = options
     end
 
@@ -41,6 +42,14 @@ module Masamune::Transform
       @presenters.key?(target_store.try(:type)) ? @presenters[target_store.try(:type)].new(@target) : @target
     end
 
+    def helper
+      (@helper || SimpleDelegator).new(self)
+    end
+
+    def locals
+      @locals
+    end
+
     def to_s
       result = []
       @templates.each do |template|
@@ -76,7 +85,7 @@ module Masamune::Transform
     def template_eval(template)
       return File.read(template) if File.exists?(template.to_s) && template.to_s !~ /erb\Z/
       template_file = File.exists?(template.to_s) ? template : template_file(template)
-      Masamune::Template.render_to_string(template_file, @locals.merge(source: source, target: target))
+      Masamune::Template.render_to_string(template_file, @locals.merge(source: source, target: target, helper: helper))
     end
 
     def template_file(template_prefix)

data/lib/masamune/transform/replace_table.psql.erb

@@ -43,12 +43,8 @@ DROP INDEX IF EXISTS <%= index_name %>;
 ALTER TABLE <%= target.name %> RENAME TO <%= target_tmp.name %>;
 ALTER TABLE <%= source.name %> RENAME TO <%= target.name %>;
 
-<%- if target.parent -%>
-ALTER TABLE <%= target.name %> INHERIT <%= target.parent.name %>;
-<%- end -%>
-<%- if target.constraints -%>
-ALTER TABLE <%= target.name %> ADD CONSTRAINT <%= target.name %>_time_key_check <%= target.constraints %>;
-<%- end -%>
+<%= render 'define_inheritance.psql.erb', target: target %>
+
 <%= render 'define_foreign_key.psql.erb', target: target, skip_check_exist: true, skip_check_valid: true %>
 
 <%= render 'define_index.psql.erb', target: target, skip_check_exist: true %>

data/lib/masamune/version.rb

@@ -21,5 +21,5 @@
 #  THE SOFTWARE.
 
 module Masamune
-  VERSION = '0.13.0'
+  VERSION = '0.13.1'
 end

data/spec/masamune/commands/postgres_admin_spec.rb

@@ -54,6 +54,21 @@ describe Masamune::Commands::PostgresAdmin do
       it { is_expected.to eq(['dropdb', '--if-exists', '--host=localhost', '--username=postgres', '--no-password', 'zombo']) }
     end
 
+    context 'action :drop with database and :output' do
+      let(:attrs) { {action: :drop, database: 'zombo', output: 'zombo.csv'} }
+      it { is_expected.to eq(['dropdb', '--if-exists', '--host=localhost', '--username=postgres', '--no-password', 'zombo']) }
+    end
+
+    context 'action :dump with database' do
+      let(:attrs) { {action: :dump, database: 'zombo'} }
+      it { is_expected.to eq(['pg_dump', '--no-owner', '--no-privileges', '--oids', '--schema=public', '--host=localhost', '--username=postgres', '--no-password', '--dbname=zombo']) }
+    end
+
+    context 'action :dump with database and :output' do
+      let(:attrs) { {action: :dump, database: 'zombo', output: 'zombo.csv'} }
+      it { is_expected.to eq(['pg_dump', '--no-owner', '--no-privileges', '--oids', '--schema=public', '--host=localhost', '--username=postgres', '--no-password', '--dbname=zombo', '--file=zombo.csv']) }
+    end
+
     context 'action :drop without database' do
       let(:attrs) { {action: :drop} }
       it { expect { subject }.to raise_error ArgumentError, ':database must be given' }
@@ -61,7 +76,7 @@ describe Masamune::Commands::PostgresAdmin do
 
     context 'action unfuddle with database' do
       let(:attrs) { {action: :unfuddle, database: 'zombo'} }
-      it { expect { subject }.to raise_error ArgumentError, ':action must be :create or :drop' }
+      it { expect { subject }.to raise_error ArgumentError, ':action must be :create, :drop, or :dump' }
     end
   end
 

data/spec/masamune/data_plan/engine_spec.rb

@@ -177,7 +177,7 @@ describe Masamune::DataPlan::Engine do
     context 'invalid target' do
       let(:rule) { 'derived_daily' }
       let(:target) {  '/table/y=2013/m=01/d=01' }
-      it { expect { subject }.to raise_error }
+      it { expect { subject }.to raise_error /Cannot bind_input/ }
     end
   end
 
@@ -206,7 +206,7 @@ describe Masamune::DataPlan::Engine do
 
     context 'invalid target' do
       let(:target) { '/daily' }
-      it { expect { subject }.to raise_error }
+      it { expect { subject }.to raise_error /No rule matches/ }
     end
   end
 

data/spec/masamune/data_plan/rule_spec.rb

@@ -47,30 +47,45 @@ describe Masamune::DataPlan::Rule do
     end
   end
 
-  describe '#bind_date' do
-    subject(:elem) { instance.bind_date(input_date) }
+  describe '#bind_date_or_time' do
+    subject(:elem) { instance.bind_date_or_time(input) }
 
-    context 'with default' do
-      let(:input_date) { DateTime.civil(2013,04,05,23,13) }
+    context 'with nil input' do
+      let(:input) { nil }
+      it { expect { elem }.to raise_error ArgumentError }
+    end
+
+    context 'with unknown input type' do
+      let(:input) { 1 }
+      it { expect { elem }.to raise_error ArgumentError }
+    end
+
+    context 'with DateTime input' do
+      let(:input) { DateTime.civil(2013,04,05,23,13) }
 
       describe '#path' do
         subject { elem.path }
         it { is_expected.to eq('report/2013-04-05/23') }
       end
-      let(:start_time) { DateTime.civil(2013,04,05,23) }
-      let(:stop_time) { DateTime.civil(2013,04,05,0) }
     end
 
-    context 'with unix timestamp pattern' do
+    context 'with DateTime input and unix timestamp pattern' do
       let(:pattern) { 'logs/%H-s.log' }
-      let(:input_date) { DateTime.civil(2013,04,05,23,13) }
+      let(:input) { DateTime.civil(2013,04,05,23,13) }
 
       describe '#path' do
         subject { elem.path }
         it { is_expected.to eq('logs/1365202800.log') }
       end
-      let(:start_time) { DateTime.civil(2013,04,05,23) }
-      let(:stop_time) { DateTime.civil(2013,04,05,0) }
+    end
+
+    context 'with Date input' do
+      let(:input) { Date.civil(2013,04,05) }
+
+      describe '#path' do
+        subject { elem.path }
+        it { is_expected.to eq('report/2013-04-05/00') }
+      end
     end
   end
 

data/spec/masamune/schema/fact_spec.rb

@@ -40,7 +40,18 @@ describe Masamune::Schema::Fact do
       ]
   end
 
-  let(:fact) do
+  let(:fact_without_partition) do
+    described_class.new id: 'visits', store: store,
+      references: [
+        Masamune::Schema::TableReference.new(date_dimension),
+        Masamune::Schema::TableReference.new(user_dimension)
+      ],
+      columns: [
+        Masamune::Schema::Column.new(id: 'total', type: :integer)
+      ]
+  end
+
+  let(:fact_with_partition) do
     described_class.new id: 'visits', store: store, partition: 'y%Ym%m',
       references: [
         Masamune::Schema::TableReference.new(date_dimension),
@@ -53,36 +64,22 @@ describe Masamune::Schema::Fact do
       ]
   end
 
-  it { expect(fact.name).to eq('visits_fact') }
-
-  describe '#partition_table' do
-    let(:date) { Chronic.parse('2015-01-01') }
-
-    subject(:partition_table) { fact.partition_table(date) }
-
-    it { expect(partition_table.store.id).to eq(store.id) }
-    it { expect(partition_table.name).to eq('visits_fact_y2015m01') }
-    it { expect(partition_table.range.start_date).to eq(date.utc.to_date) }
-
-    describe '#stage_table' do
-      subject(:stage_table) { partition_table.stage_table }
-
-      it { expect(stage_table.store.id).to eq(store.id) }
-      it { expect(stage_table.name).to eq('visits_fact_y2015m01_stage') }
-      it { expect(stage_table.range.start_date).to eq(date.utc.to_date) }
-
-      context 'with optional suffix' do
-        subject(:stage_table) { partition_table.stage_table(suffix: 'tmp') }
-
-        it 'should append suffix to id' do
-          expect(stage_table.store.id).to eq(store.id)
-          expect(stage_table.name).to eq('visits_fact_y2015m01_stage_tmp')
-          expect(stage_table.range.start_date).to eq(date.utc.to_date)
-        end
-      end
-    end
+  let(:fact_with_partition_and_hourly_grain) do
+    described_class.new id: 'visits', store: store, grain: :hourly, partition: 'y%Ym%m',
+      references: [
+        Masamune::Schema::TableReference.new(date_dimension),
+        Masamune::Schema::TableReference.new(user_dimension)
+      ],
+      columns: [
+        Masamune::Schema::Column.new(id: 'total', type: :integer)
+      ]
   end
 
+  it { expect(fact_without_partition.name).to eq('visits_fact') }
+  it { expect(fact_with_partition.name).to eq('visits_fact') }
+  it { expect(fact_with_partition_and_hourly_grain.id).to eq(:visits_hourly) }
+  it { expect(fact_with_partition_and_hourly_grain.name).to eq('visits_hourly_fact') }
+
   context 'fact with unknown grain' do
     subject(:fact) do
       described_class.new id: 'visits', grain: :quarterly
@@ -91,25 +88,44 @@ describe Masamune::Schema::Fact do
     it { expect { fact }.to raise_error ArgumentError, "unknown grain 'quarterly'" }
   end
 
-  context 'fact with :hourly grain' do
-    let(:fact) do
-      described_class.new id: 'visits', store: store, grain: :hourly, partition: 'y%Ym%m',
-        references: [
-          Masamune::Schema::TableReference.new(date_dimension),
-          Masamune::Schema::TableReference.new(user_dimension)
-        ],
-        columns: [
-          Masamune::Schema::Column.new(id: 'total', type: :integer)
-        ]
+  describe '#partition_table' do
+    let(:date) { Chronic.parse('2015-01-01') }
+    subject(:partition_table) { fact.partition_table(date) }
+
+    context 'fact without partition' do
+      let(:fact) { fact_without_partition }
+
+      it { expect(partition_table).to be_nil }
     end
 
-    it { expect(fact.id).to eq(:visits_hourly) }
-    it { expect(fact.name).to eq('visits_hourly_fact') }
+    context 'fact with partition' do
+      let(:fact) { fact_with_partition }
+
+      it { expect(partition_table.store.id).to eq(store.id) }
+      it { expect(partition_table.name).to eq('visits_fact_y2015m01') }
+      it { expect(partition_table.range.start_date).to eq(date.utc.to_date) }
+
+      describe '#stage_table' do
+        subject(:stage_table) { partition_table.stage_table }
+
+        it { expect(stage_table.store.id).to eq(store.id) }
+        it { expect(stage_table.name).to eq('visits_fact_y2015m01_stage') }
+        it { expect(stage_table.range.start_date).to eq(date.utc.to_date) }
+
+        context 'with optional suffix' do
+          subject(:stage_table) { partition_table.stage_table(suffix: 'tmp') }
 
-    describe '#partition_table' do
-      let(:date) { Chronic.parse('2015-01-01') }
+          it 'should append suffix to id' do
+            expect(stage_table.store.id).to eq(store.id)
+            expect(stage_table.name).to eq('visits_fact_y2015m01_stage_tmp')
+            expect(stage_table.range.start_date).to eq(date.utc.to_date)
+          end
+        end
+      end
+    end
 
-      subject(:partition_table) { fact.partition_table(date) }
+    context 'fact with partition and hourly grain' do
+      let(:fact) { fact_with_partition_and_hourly_grain }
 
       it { expect(partition_table.store.id).to eq(store.id) }
       it { expect(partition_table.name).to eq('visits_hourly_fact_y2015m01') }
@@ -126,4 +142,28 @@ describe Masamune::Schema::Fact do
       end
     end
   end
+
+  describe '#partition_tables' do
+    let(:start_date) { Date.civil(2015, 01, 01) }
+    let(:stop_date) { Date.civil(2015, 03, 15) }
+
+    subject(:partition_tables) { fact.partition_tables(start_date, stop_date) }
+
+    context 'fact without partition' do
+      let(:fact) { fact_without_partition }
+
+      it { expect(partition_tables).to be_nil }
+    end
+
+    context 'fact with partition' do
+      let(:fact) { fact_with_partition }
+
+      it 'yields partition tables' do
+        expect { |b| fact.partition_tables(start_date, stop_date, &b) }.to yield_successive_args \
+           fact.partition_table(Date.civil(2015, 01, 01)),
+           fact.partition_table(Date.civil(2015, 02, 01)),
+           fact.partition_table(Date.civil(2015, 03, 01))
+      end
+    end
+  end
 end

data/spec/masamune/tasks/dump_thor_spec.rb

@@ -32,11 +32,51 @@ describe Masamune::Tasks::DumpThor do
   end
 
   context 'with no arguments' do
-    it 'exits with status code 0 and prints catalog' do
-      expect { cli_invocation }.to raise_error { |e|
-        expect(e).to be_a(SystemExit)
-        expect(e.status).to eq(0)
-      }
-    end
+    it_behaves_like 'executes with success'
+  end
+
+  context 'with --type=psql' do
+    let(:options) { ['--type=psql'] }
+    it_behaves_like 'executes with success'
+  end
+
+  context 'with --type=hql' do
+    let(:options) { ['--type=hql'] }
+    it_behaves_like 'executes with success'
+  end
+
+  context 'with --type=unknown' do
+    let(:options) { ['--type=unknown'] }
+    it_behaves_like 'raises Thor::MalformattedArgumentError', %q{Expected '--type' to be one of psql, hql; got unknown}
+  end
+
+  context 'with --section=pre' do
+    let(:options) { ['--section=pre'] }
+    it_behaves_like 'executes with success'
+  end
+
+  context 'with --section=post' do
+    let(:options) { ['--section=post'] }
+    it_behaves_like 'executes with success'
+  end
+
+  context 'with --section=all' do
+    let(:options) { ['--section=all'] }
+    it_behaves_like 'executes with success'
+  end
+
+  context 'with --section=unknown' do
+    let(:options) { ['--section=unknown'] }
+    it_behaves_like 'raises Thor::MalformattedArgumentError', %q{Expected '--section' to be one of pre, post, all; got unknown}
+  end
+
+  context 'with --start=yesterday' do
+    let(:options) { ['--start=yesterday'] }
+    it_behaves_like 'executes with success'
+  end
+
+  context 'with --stop=today' do
+    let(:options) { ['--stop=today'] }
+    it_behaves_like 'executes with success'
   end
 end

data/spec/masamune/thor_spec.rb

@@ -99,43 +99,44 @@ describe Masamune::Thor do
 
     context 'with command and no input options' do
       let(:command) { 'command' }
-      it { expect { cli_invocation }.to raise_error Thor::RequiredArgumentMissingError, /No value provided for required options '--start'/ }
+      it_behaves_like 'raises Thor::RequiredArgumentMissingError', /No value provided for required options '--start'/
     end
 
     context 'with command and invalid --start' do
       let(:command) { 'command' }
       let(:options) { ['--start', 'xxx'] }
-      it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected date time value for '--start'; got/ }
+      it_behaves_like 'raises Thor::MalformattedArgumentError', /Expected date time value for '--start'; got/
     end
 
     context 'with command and invalid --stop' do
       let(:command) { 'command' }
       let(:options) { ['--start', '2013-01-01', '--stop', 'xxx'] }
-      it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected date time value for '--stop'; got/ }
+      it_behaves_like 'raises Thor::MalformattedArgumentError', /Expected date time value for '--stop'; got/
     end
 
     context 'with command and invalid --sources' do
       let(:command) { 'command' }
       let(:options) { ['--sources', 'foo'] }
-      it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected file value for '--sources'; got/ }
+      it_behaves_like 'raises Thor::MalformattedArgumentError', /Expected file value for '--sources'; got/
     end
 
     context 'with command and invalid --targets' do
       let(:command) { 'command' }
       let(:options) { ['--targets', 'foo'] }
-      it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected file value for '--targets'; got/ }
+      it_behaves_like 'raises Thor::MalformattedArgumentError', /Expected file value for '--targets'; got/
     end
 
     context 'with command and both --sources and --targets' do
       let(:command) { 'command' }
       let(:options) { ['--sources', 'sources', '--targets', 'targets'] }
-      it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Cannot specify both option '--sources' and option '--targets'/ }
+
+      it_behaves_like 'raises Thor::MalformattedArgumentError', /Cannot specify both option '--sources' and option '--targets'/
     end
 
     context 'with command and --start and bad --config file' do
       let(:command) { 'command' }
       let(:options) { ['--start', '2013-01-01', '--config', 'xxx'] }
-      it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Could not load file provided for '--config'/ }
+      it_behaves_like 'raises Thor::MalformattedArgumentError', /Could not load file provided for '--config'/
     end
 
     context 'with command and --start and missing system --config file' do
@@ -144,7 +145,7 @@ describe Masamune::Thor do
       before do
         expect_any_instance_of(Masamune::Filesystem).to receive(:resolve_file)
       end
-      it { expect { cli_invocation }.to raise_error Thor::RequiredArgumentMissingError, /Option --config or valid system configuration file required/ }
+      it_behaves_like 'raises Thor::RequiredArgumentMissingError', /Option --config or valid system configuration file required/
     end
 
     context 'with command and -- --extra --args' do
@@ -153,35 +154,19 @@ describe Masamune::Thor do
       before do
         expect_any_instance_of(thor_class).to receive(:extra=).with(['--extra', '--args'])
       end
-      it do
-        expect { cli_invocation }.to raise_error SystemExit
-      end
+      it_behaves_like 'executes with success'
     end
 
     context 'with command and --start' do
       let(:command) { 'command' }
       let(:options) { ['--start', '2013-01-01'] }
-      it 'exits with status code 0 without error message' do
-        expect { cli_invocation }.to raise_error { |e|
-          expect(e).to be_a(SystemExit)
-          expect(e.status).to eq(0)
-        }
-        expect(stdout.string).to match(/\AUsing '.*' for --start/)
-        expect(stderr.string).to eq('')
-      end
+      it_behaves_like 'executes with success'
     end
 
     context 'with command and natural language --start' do
       let(:command) { 'command' }
       let(:options) { ['--start', 'yesterday'] }
-      it 'exits with status code  0 without error message' do
-        expect { cli_invocation }.to raise_error { |e|
-          expect(e).to be_a(SystemExit)
-          expect(e.status).to eq(0)
-        }
-        expect(stdout.string).to match(/\AUsing '.*' for --start/)
-        expect(stderr.string).to eq('')
-      end
+      it_behaves_like 'executes with success'
     end
 
     context 'with command that raises exception before initialization' do