masamune 0.17.4 → 0.17.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 61f6f61c4cd5df082a7f5a63cee2f4f7716ff7ca
-  data.tar.gz: 9a47bb9bbdb0339904cf8159968fe9fc0a6dc6aa
+  metadata.gz: 3067d46a85020fb673687d897550d3590dd34e15
+  data.tar.gz: 8f3a034427ba96b5209434cb997d1eb7c313e91d
 SHA512:
-  metadata.gz: e7bd7a01c74b0779318dc2988c1741fed4122c866b913fe63457d4696a47794ffb540de0d363bb8d92b9f4836de3be1ed929dc741a90085c093b216728e8aaae
-  data.tar.gz: 5ec032b8c286a99d8873a03525ba6c02c2f090567e7a451711423b6e2fd2f6d87133deabafcd527dd149b502b1c5c07ff03bff6b24037bd202a44143ded3cd1d
+  metadata.gz: a061bfc2c774867f1430534b0974f7d76ee4f8f5eeba0d6a796d698c7779b7f2c19d95996a526d1b915b183813c30f160a159cb96505079af9b205e2805a8631
+  data.tar.gz: 21f0072fa6b117d8525302064458c3ce987ede2947ef2138a45f0840cb46701b767e6d86cf12812b01c420f0c1097de8f9db66846571b5903acccff8c5f58c1a

data/lib/masamune/tasks/dump_thor.rb

@@ -37,6 +37,7 @@ module Masamune::Tasks
     desc 'dump', 'Dump schema'
     method_option :type, :enum => ['psql', 'hql'], :desc => 'Schema type', :default => 'psql'
     method_option :section, :enum => ['pre', 'post', 'all'], :desc => 'Schema section', :default => 'all'
+    method_option :exclude, :type => :array, :desc => 'Exclude tables matching globs', :default => []
     def dump_exec
       print_catalog
       exit
@@ -56,6 +57,7 @@ module Masamune::Tasks
 
     def define_schema_options
       {
+        exclude: options[:exclude],
         section: options[:section].to_sym,
         start_date: start_date,
         stop_date: stop_date

data/lib/masamune/transform/define_schema.rb

@@ -35,13 +35,13 @@ module Masamune::Transform
       operators += context.extra(:pre)
 
       context.dimensions.each do |_, dimension|
-        operators << define_table(dimension, [], options[:section])
+        operators << define_table(dimension, options)
       end
 
       context.facts.each do |_, fact|
-        operators << define_table(fact, [], options[:section])
+        operators << define_table(fact, options)
         fact.partition_tables(options[:start_date], options[:stop_date]) do |fact_partition_table|
-          operators << define_table(fact_partition_table, [], options[:section])
+          operators << define_table(fact_partition_table, options)
         end
       end
 

data/lib/masamune/transform/define_table.rb

@@ -24,11 +24,38 @@ module Masamune::Transform
   module DefineTable
     extend ActiveSupport::Concern
 
-    def define_table(target, files = [], section = nil)
+    def define_table(target, options = {})
       return if target.implicit
-      Operator.new(__method__, target: target, files: files, section: section).tap do |operator|
+      return if exclude_table?(target, options)
+      child_tables = target.children.map { |child| define_table(child, options.except(:files)) }
+      Operator.new(*child_tables, __method__, target: target, files: options[:files], section: options[:section]).tap do |operator|
         logger.debug("#{target.id}\n" + operator.to_s) if target.debug
       end
     end
+
+    private
+
+    def exclude_table?(table, options = {})
+      exclude_matchers(options[:exclude]).any? { |matcher| matcher =~ table.name }
+    end
+
+    def exclude_matchers(exclude)
+      Array.wrap(exclude).map do |input|
+        case input
+        when String
+          glob_to_regexp(input)
+        when Regexp
+          input
+        end
+      end
+    end
+
+    def glob_to_regexp(input)
+      if input.include?('*')
+        %r|\A#{Regexp.escape(input).gsub('\\*', '.*?')}|
+      else
+        /\A#{Regexp.escape(input)}\z/
+      end
+    end
   end
 end

data/lib/masamune/transform/load_dimension.rb

@@ -38,7 +38,7 @@ module Masamune::Transform
       target = target.type == :four ? target.ledger_table : target
       source = source.stage_table(suffix: 'file', table: target, inherit: false)
       Operator.new \
-        define_table(source, files),
+        define_table(source, files: files),
         insert_reference_values(source, target),
         stage_dimension(source, target),
         bulk_upsert(target.stage_table, target)

data/lib/masamune/transform/load_fact.rb

@@ -37,7 +37,7 @@ module Masamune::Transform
     def load_fact(files, source, target, date)
       source = source.stage_table(suffix: 'file', table: target, inherit: false)
       Operator.new \
-        define_table(source, files),
+        define_table(source, files: files),
         insert_reference_values(source, target),
         stage_fact(source, target, date)
     end

data/lib/masamune/transform/postgres/define_table.psql.erb

@@ -24,10 +24,6 @@
   files ||= []
 %>
 
-<%- target.children.each do |child| -%>
-<%= render 'define_table.psql.erb', target: child, **locals.except(:target, :files)  %>
-<%- end -%>
-
 <%- if helper.define_types? %>
 <%- target.enum_columns.each do |_, column| -%>
 DO $$ BEGIN

data/lib/masamune/transform/postgres/define_table.rb

@@ -90,7 +90,7 @@ module Masamune::Transform::Postgres
     end
 
     def insert_rows?
-      !post_section?
+      !pre_section? && !post_section?
     end
 
     def load_files?
@@ -118,10 +118,6 @@ module Masamune::Transform::Postgres
     end
 
     class TargetPresenter < SimpleDelegator
-      def children
-        super.map { |child| self.class.new(child) }
-      end
-
       def inherited?
         type == :fact && inheritance_constraints
       end

data/lib/masamune/version.rb

@@ -21,5 +21,5 @@
 #  THE SOFTWARE.
 
 module Masamune
-  VERSION = '0.17.4'
+  VERSION = '0.17.5'
 end

data/spec/masamune/tasks/dump_thor_spec.rb

@@ -67,6 +67,16 @@ describe Masamune::Tasks::DumpThor do
     it_behaves_like 'raises Thor::MalformattedArgumentError', %q{Expected '--section' to be one of pre, post, all; got unknown}
   end
 
+  context %q{with --exclude='.*dimension'} do
+    let(:options) { [%q{--exclude='.*dimension'}] }
+    it_behaves_like 'executes with success'
+  end
+
+  context %q{with --exclude='.*dimension' '.*fact'} do
+    let(:options) { [%q{--exclude='.*dimension' '.*fact'}] }
+    it_behaves_like 'executes with success'
+  end
+
   context 'with --start=yesterday' do
     let(:options) { ['--start=yesterday'] }
     it_behaves_like 'executes with success'

data/spec/masamune/transform/define_schema_spec.rb

@@ -66,7 +66,7 @@ describe Masamune::Transform::DefineSchema do
       end
     end
 
-    context 'without start_date and stop_date' do
+    context 'with start_date and stop_date' do
       subject(:result) { transform.define_schema(catalog, :postgres, start_date: Date.civil(2015, 01, 01), stop_date: Date.civil(2015, 03, 15)).to_s }
 
       it 'should render combined template' do
@@ -80,6 +80,28 @@ describe Masamune::Transform::DefineSchema do
           transform.define_table(catalog.postgres.facts['visits'].partition_table(Date.civil(2015, 03, 01)))
       end
     end
+
+    context 'with exclude Regexp' do
+      subject(:result) { transform.define_schema(catalog, :postgres, exclude: /.*dimension/).to_s }
+
+      it 'should render combined template' do
+        is_expected.to eq Masamune::Template.combine \
+          Masamune::Transform::Operator.new('define_schema', source: catalog.postgres),
+          transform.define_table(catalog.postgres.dimensions['user_account_state']),
+          transform.define_table(catalog.postgres.facts['visits'])
+      end
+    end
+
+    context 'with exclude glob' do
+      subject(:result) { transform.define_schema(catalog, :postgres, exclude: '*dimension').to_s }
+
+      it 'should render combined template' do
+        is_expected.to eq Masamune::Template.combine \
+          Masamune::Transform::Operator.new('define_schema', source: catalog.postgres),
+          transform.define_table(catalog.postgres.dimensions['user_account_state']),
+          transform.define_table(catalog.postgres.facts['visits'])
+      end
+    end
   end
 
   context 'for hive schema' do

data/spec/masamune/transform/define_table.dimension_spec.rb

@@ -21,7 +21,8 @@
 #  THE SOFTWARE.
 
 describe Masamune::Transform::DefineTable do
-  subject { transform.define_table(table).to_s }
+  let(:options) { {} }
+  subject { transform.define_table(table, options).to_s }
 
   context 'for hive implicit dimension' do
     before do
@@ -415,4 +416,70 @@ describe Masamune::Transform::DefineTable do
       EOS
     end
   end
+
+  context 'for postgres dimension type: four with exclude /.*ledger/' do
+    before do
+      catalog.schema :postgres do
+        dimension 'user', type: :four do
+          column 'tenant_id', natural_key: true
+          column 'user_id', natural_key: true
+          column 'preferences', type: :key_value, null: true
+        end
+      end
+    end
+
+    let(:options) { { exclude: /.*ledger\z/ } }
+    let(:table) { catalog.postgres.user_dimension }
+
+    it 'should render table template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+        CREATE TABLE IF NOT EXISTS user_dimension
+        (
+          id SERIAL,
+          tenant_id INTEGER NOT NULL,
+          user_id INTEGER NOT NULL,
+          preferences HSTORE,
+          start_at TIMESTAMP NOT NULL DEFAULT TO_TIMESTAMP(0),
+          end_at TIMESTAMP,
+          version INTEGER DEFAULT 1,
+          last_modified_at TIMESTAMP NOT NULL DEFAULT NOW()
+        );
+
+        DO $$ BEGIN
+        IF NOT EXISTS (SELECT 1 FROM pg_class c WHERE c.relname = 'user_dimension_pkey') THEN
+        ALTER TABLE user_dimension ADD PRIMARY KEY (id);
+        END IF; END $$;
+
+        DO $$ BEGIN
+        IF NOT EXISTS (SELECT 1 FROM pg_class c WHERE c.relname = 'user_dimension_e6c3d91_key') THEN
+        ALTER TABLE user_dimension ADD CONSTRAINT user_dimension_e6c3d91_key UNIQUE(tenant_id, user_id, start_at);
+        END IF; END $$;
+
+        DO $$ BEGIN
+        IF NOT EXISTS (SELECT 1 FROM pg_class c WHERE c.relname = 'user_dimension_2c8e908_index') THEN
+        CREATE INDEX user_dimension_2c8e908_index ON user_dimension (end_at);
+        END IF; END $$;
+
+        DO $$ BEGIN
+        IF NOT EXISTS (SELECT 1 FROM pg_class c WHERE c.relname = 'user_dimension_23563d3_index') THEN
+        CREATE INDEX user_dimension_23563d3_index ON user_dimension (start_at);
+        END IF; END $$;
+
+        DO $$ BEGIN
+        IF NOT EXISTS (SELECT 1 FROM pg_class c WHERE c.relname = 'user_dimension_3854361_index') THEN
+        CREATE INDEX user_dimension_3854361_index ON user_dimension (tenant_id);
+        END IF; END $$;
+
+        DO $$ BEGIN
+        IF NOT EXISTS (SELECT 1 FROM pg_class c WHERE c.relname = 'user_dimension_e8701ad_index') THEN
+        CREATE INDEX user_dimension_e8701ad_index ON user_dimension (user_id);
+        END IF; END $$;
+
+        DO $$ BEGIN
+        IF NOT EXISTS (SELECT 1 FROM pg_class c WHERE c.relname = 'user_dimension_e6c3d91_index') THEN
+        CREATE UNIQUE INDEX user_dimension_e6c3d91_index ON user_dimension (tenant_id, user_id, start_at);
+        END IF; END $$;
+      EOS
+    end
+  end
 end

data/spec/masamune/transform/define_table.fact_spec.rb

@@ -122,11 +122,12 @@ describe Masamune::Transform::DefineTable do
     end
   end
 
+  let(:options) { {} }
+  subject(:result) { transform.define_table(target, options).to_s }
+
   context 'for postgres fact' do
     let(:target) { catalog.postgres.visits_fact }
 
-    subject(:result) { transform.define_table(target).to_s }
-
     it 'should eq render table template' do
       is_expected.to eq <<-EOS.strip_heredoc
         CREATE TABLE IF NOT EXISTS visits_fact
@@ -149,7 +150,7 @@ describe Masamune::Transform::DefineTable do
   context 'for postgres fact partition with :post' do
     let(:target) { catalog.postgres.visits_fact.partition_table(Date.civil(2015, 01, 01)) }
 
-    subject(:result) { transform.define_table(target, [], :post).to_s }
+    let(:options)  { {section: :post} }
 
     it 'should eq render table template' do
       is_expected.to match /ALTER TABLE visits_fact_y2015m01 INHERIT visits_fact;/
@@ -159,10 +160,10 @@ describe Masamune::Transform::DefineTable do
 
   describe 'for fact table from file with sources files' do
     let(:files) { (1..3).map { |i| double(path: "output_#{i}.csv") } }
-    let(:target) { catalog.postgres.visits_fact }
+    let(:intermediate) { catalog.postgres.visits_fact }
     let(:source) { catalog.postgres.visits_file }
-
-    subject(:result) { transform.define_table(source.stage_table(suffix: 'file', table: target, inherit: false), files).to_s }
+    let(:target) { source.stage_table(suffix: 'file', table: intermediate, inherit: false) }
+    let(:options) { { files: files} }
 
     it 'should eq render table template' do
       is_expected.to eq <<-EOS.strip_heredoc
@@ -200,15 +201,19 @@ describe Masamune::Transform::DefineTable do
       EOS
     end
 
-    context 'with file' do
-      subject(:result) { transform.define_table(source.stage_table(table: target), files.first).to_s }
+    context 'with single file' do
+      let(:target) { source.stage_table(table: intermediate) }
+      let(:options) { { files: files.first } }
+
       it 'should eq render table template' do
         is_expected.to_not be_nil
       end
     end
 
     context 'with Set' do
-      subject(:result) { transform.define_table(source.stage_table(table: target), Set.new(files)).to_s }
+      let(:target) { source.stage_table(table: intermediate) }
+      let(:options) { { files: Set.new(files) } }
+
       it 'should eq render table template' do
         is_expected.to_not be_nil
       end
@@ -228,8 +233,6 @@ describe Masamune::Transform::DefineTable do
 
     let(:target) { catalog.postgres.visits_fact }
 
-    subject(:result) { transform.define_table(target).to_s }
-
     it 'should eq render table template' do
       is_expected.to eq <<-EOS.strip_heredoc
         CREATE TABLE IF NOT EXISTS visits_fact
@@ -246,8 +249,6 @@ describe Masamune::Transform::DefineTable do
   context 'for hive fact' do
     let(:target) { catalog.hive.visits_hourly_fact }
 
-    subject(:result) { transform.define_table(target).to_s }
-
     it 'should eq render table template' do
       is_expected.to eq <<-EOS.strip_heredoc
         CREATE TABLE IF NOT EXISTS visits_hourly_fact

data/spec/masamune/transform/define_table.table_spec.rb

@@ -23,8 +23,9 @@
 describe Masamune::Transform::DefineTable do
   let(:files) { [] }
   let(:section) { :all }
+  let(:options) { { files: files, section: section } }
 
-  subject { transform.define_table(target, files, section).to_s }
+  subject { transform.define_table(target, options).to_s }
 
   context 'for postgres table with columns' do
     before do
@@ -875,6 +876,39 @@ describe Masamune::Transform::DefineTable do
     end
   end
 
+  context 'for postgres table with row data and section :pre' do
+    before do
+      catalog.schema :postgres do
+        table 'user_account_state' do
+          column 'name', type: :string, unique: true
+          column 'description', type: :string
+          row name: 'registered', description: 'Registered'
+          row name: 'active', description: 'Active', attributes: {default: true}
+          row name: 'inactive', description: 'Inactive'
+        end
+      end
+    end
+
+    let(:section) { :pre }
+    let(:target) { catalog.postgres.user_account_state_table }
+
+    it 'should render table template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+        CREATE TABLE IF NOT EXISTS user_account_state_table
+        (
+          id SERIAL,
+          name VARCHAR NOT NULL,
+          description VARCHAR NOT NULL
+        );
+
+        CREATE OR REPLACE FUNCTION default_user_account_state_table_id()
+        RETURNS INTEGER IMMUTABLE AS $$
+          SELECT id FROM user_account_state_table WHERE name = 'active' AND description = 'Active';
+        $$ LANGUAGE SQL;
+      EOS
+    end
+  end
+
   context 'for postgres table with unique columns and section :post' do
     before do
       catalog.schema :postgres do

data/spec/masamune/transform/load_dimension_spec.rb

@@ -54,18 +54,18 @@ describe Masamune::Transform::LoadDimension do
     end
   end
 
-  let(:data) { double(path: 'output.csv') }
+  let(:files) { double(path: 'output.csv') }
   let(:target) { catalog.postgres.user_dimension }
   let(:source) { catalog.postgres.user_file }
   let(:target_ledger) { target.ledger_table }
   let(:source_table) { source.stage_table(suffix: 'file', table: target_ledger, inherit: false) }
 
   context 'with postgres dimension' do
-    subject(:result) { transform.load_dimension(data, source, target).to_s }
+    subject(:result) { transform.load_dimension(files, source, target).to_s }
 
     it 'should render combined template' do
       is_expected.to eq Masamune::Template.combine \
-        transform.define_table(source_table, data),
+        transform.define_table(source_table, files: files),
         transform.insert_reference_values(source_table, target_ledger),
         transform.stage_dimension(source_table, target_ledger),
         transform.bulk_upsert(target_ledger.stage_table, target_ledger)

data/spec/masamune/transform/load_fact_spec.rb

@@ -79,7 +79,7 @@ describe Masamune::Transform::LoadFact do
 
     it 'should render combined template' do
       is_expected.to eq Masamune::Template.combine \
-        transform.define_table(source_table, files),
+        transform.define_table(source_table, files: files),
         transform.insert_reference_values(source_table, target),
         transform.stage_fact(source_table, target, date)
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: masamune
 version: !ruby/object:Gem::Version
-  version: 0.17.4
+  version: 0.17.5
 platform: ruby
 authors:
 - Michael Andrews
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-08 00:00:00.000000000 Z
+date: 2016-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor