masamune 0.13.0 → 0.13.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 77bd1d9e8351f259a7d946ac7b2abd11b4c14408
-  data.tar.gz: deb47f8efaadd8b90b5825f79970e46833bbb030
+  metadata.gz: 49fdc180c59d4179763efd965d946c93fb864dcf
+  data.tar.gz: 0765d61a7a7247c7bb5eb0d70d17c7edb5e5fcf5
 SHA512:
-  metadata.gz: 784416ce6169e2c3f2a9a14465580417ecfc6bdd73e238643022f608a9a04ede66d0b670dfce06e0956b14d3d96a8edbcbe0018ee76bdd0e1ccd4c2bb9f4b37a
-  data.tar.gz: 6b1cb22550d392cb6bf6cf36258d21b28e7c323dd6bab5616b4066380e315bcb289c725d52a36f65cd182293c1a28d68f31e928a642d5429ce1f024945f2d81a
+  metadata.gz: 7a121d38a8d42a7e0c255ba17e523e1520da7016e0a87d553adcd32c67f6af21eb2ffc81cbcda1123e286fd93cdde8b31177affa30e17314497248d820fea989
+  data.tar.gz: 93ab83f1f5804d0b3bf25970a665d63f3e7099c43ec6c817e8a04ee30ad3dde348aa4a2515a1a8518e76aeb0150eed04a77cf1a568710f806dbe4582f1502259

data/lib/masamune/actions/date_parse.rb

@@ -30,7 +30,7 @@ module Masamune::Actions
     def parse_datetime_type(key)
       value = options[key]
       Chronic.parse(value).tap do |datetime_value|
-        console("Using '#{datetime_value}' for --#{key}") if value != datetime_value
+        logger.debug("Using '#{datetime_value}' for --#{key}") if value != datetime_value
       end or raise Thor::MalformattedArgumentError, "Expected date time value for '--#{key}'; got #{value}"
     end
 

data/lib/masamune/commands/postgres_admin.rb

@@ -30,12 +30,14 @@ module Masamune::Commands
     {
       :create_db_path => 'createdb',
       :drop_db_path   => 'dropdb',
+      :pg_dump_path   => 'pg_dump',
       :options        => [],
       :hostname       => 'localhost',
       :username       => 'postgres',
       :pgpass_file    => nil,
       :action         => nil,
-      :database       => nil
+      :database       => nil,
+      :output         => nil
     }
 
     def initialize(delegate, attrs = {})
@@ -52,7 +54,9 @@ module Masamune::Commands
       args << '--host=%s' % @hostname if @hostname
       args << '--username=%s' % @username if @username
       args << '--no-password'
-      args << @database
+      args << database
+      args << @options
+      args << output
       args.flatten.compact
     end
 
@@ -64,9 +68,21 @@ module Masamune::Commands
         [@create_db_path]
       when :drop
         [@drop_db_path, '--if-exists']
+      when :dump
+        [@pg_dump_path, '--no-owner', '--no-privileges', '--oids', '--schema=public']
       else
-        raise ArgumentError, ':action must be :create or :drop'
+        raise ArgumentError, ':action must be :create, :drop, or :dump'
       end
     end
+
+    def database
+      return @database unless @action == :dump
+      '--dbname=%s' % @database
+    end
+
+    def output
+      return unless @action == :dump
+      '--file=%s' % @output if @output
+    end
   end
 end

data/lib/masamune/data_plan/rule.rb

@@ -109,9 +109,18 @@ class Masamune::DataPlan::Rule
     matched_pattern.present? && matched_pattern[:rest].blank?
   end
 
-  def bind_date(input_date)
-    output_date = tz.utc_to_local(input_date)
-    Masamune::DataPlan::Elem.new(self, output_date, options_for_elem)
+  def bind_date_or_time(input = nil)
+    input_time =
+    case input
+    when Time, DateTime
+      input
+    when Date
+      input.to_time
+    else
+      raise ArgumentError, "Cannot bind_date_or_time with type #{input.class}"
+    end
+    output_time = tz.utc_to_local(input_time)
+    Masamune::DataPlan::Elem.new(self, output_time, options_for_elem)
   end
 
   def bind_input(input)
@@ -123,12 +132,12 @@ class Masamune::DataPlan::Rule
   end
 
   def unify(elem, rule)
-    rule.bind_date(elem.start_time)
+    rule.bind_date_or_time(elem.start_time)
   end
 
   def generate(start_time, stop_time)
     return Set.new(to_enum(:generate, start_time, stop_time)) unless block_given?
-    instance = bind_date(start_time)
+    instance = bind_date_or_time(start_time)
 
     begin
       yield instance

data/lib/masamune/schema/fact.rb

@@ -85,17 +85,28 @@ module Masamune::Schema
       end
     end
 
-    def partition_table(date)
-      partition_range = partition_rule.bind_date(date)
+    def partition_table(date = nil)
+      return unless partition
+      return unless date
+      partition_range = partition_rule.bind_date_or_time(date)
       @partition_tables ||= {}
       @partition_tables[partition_range] ||= self.class.new(id: @id, store: store, columns: partition_table_columns, parent: self, range: partition_range, grain: grain, inherit: true)
     end
 
+    def partition_tables(start_date = nil, stop_date = nil)
+      return unless partition
+      return unless start_date && stop_date
+      (start_date .. stop_date).each do |date|
+        next unless date.day == 1
+        yield partition_table(date)
+      end
+    end
+
     def measures
       columns.select { |_, column| column.measure }
     end
 
-    def constraints
+    def inheritance_constraints
       return unless range
       "CHECK (time_key >= #{range.start_time.to_i} AND time_key < #{range.stop_time.to_i})"
     end

data/lib/masamune/schema/table.rb

@@ -183,8 +183,13 @@ module Masamune::Schema
       return to_enum(__method__).to_a.flatten.compact unless block_given?
       columns.map do |_, column|
         next if column.surrogate_key || column.ignore
-        if column.reference
-          (column.reference.natural_keys.any? ? column.reference.natural_keys : column.reference.denormalized_columns).each do |join_column|
+        if column.reference && column.reference.natural_keys.any?
+          column.reference.natural_keys.each do |join_column|
+            next if join_column.reference && join_column.natural_key
+            yield [column.reference, join_column]
+          end
+        elsif column.reference && column.reference.denormalized_columns.any?
+          column.reference.denormalized_columns.each do |join_column|
             yield [column.reference, join_column]
           end
         else

data/lib/masamune/tasks/dump_thor.rb

@@ -26,6 +26,7 @@ require 'thor'
 module Masamune::Tasks
   class DumpThor < Thor
     include Masamune::Thor
+    include Masamune::Actions::DateParse
     include Masamune::Transform::DefineSchema
 
     # FIXME need to add an unnecessary namespace until this issue is fixed:
@@ -35,9 +36,7 @@ module Masamune::Tasks
 
     desc 'dump', 'Dump schema'
     method_option :type, :enum => ['psql', 'hql'], :desc => 'Schema type', :default => 'psql'
-    method_option :with_index, :type => :boolean, :desc => 'Dump schema with indexes', :default => true
-    method_option :with_foreign_key, :type => :boolean, :desc => 'Dump schema with foreign key constraints', :default => true
-    method_option :with_unique_constraint, :type => :boolean, :desc => 'Dump schema with uniqueness constraints', :default => true
+    method_option :section, :enum => ['pre', 'post', 'all'], :desc => 'Schema section', :default => 'all'
     def dump_exec
       print_catalog
       exit
@@ -49,10 +48,18 @@ module Masamune::Tasks
     def print_catalog
       case options[:type]
       when 'psql'
-        puts define_schema(catalog, :postgres, options.slice(:with_index, :with_foreign_key, :with_unique_constraint).to_h.symbolize_keys)
+        puts define_schema(catalog, :postgres, define_schema_options)
       when 'hql'
-        puts define_schema(catalog, :hive)
+        puts define_schema(catalog, :hive, define_schema_options)
       end
     end
+
+    def define_schema_options
+      {
+        section: options[:section].to_sym,
+        start_date: start_date,
+        stop_date: stop_date
+      }.reject { |_, v| v.blank? }
+    end
   end
 end

data/lib/masamune/template.rb

@@ -56,7 +56,7 @@ module Masamune
       end
 
       def render_to_string(template, parameters = {})
-        instance = Template.new(File.dirname(template))
+        instance = self.new(File.dirname(template))
         combine instance.render(template, parameters)
       end
 

data/lib/masamune/transform/define_inheritance.psql.erb

@@ -0,0 +1,28 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+<%- if target.parent -%>
+ALTER TABLE <%= target.name %> INHERIT <%= target.parent.name %>;
+<%- end -%>
+<%- if target.inheritance_constraints -%>
+ALTER TABLE <%= target.name %> ADD CONSTRAINT <%= target.name %>_time_key_check <%= target.inheritance_constraints %>;
+<%- end -%>

data/lib/masamune/transform/define_schema.rb

@@ -35,11 +35,14 @@ module Masamune::Transform
       operators += context.extra(:pre)
 
       context.dimensions.each do |_, dimension|
-        operators << define_table(dimension, [], options)
+        operators << define_table(dimension, [], options[:section])
       end
 
       context.facts.each do |_, fact|
-        operators << define_table(fact, [], options)
+        operators << define_table(fact, [], options[:section])
+        fact.partition_tables(options[:start_date], options[:stop_date]) do |fact_partition_table|
+          operators << define_table(fact_partition_table, [], options[:section])
+        end
       end
 
       operators += context.extra(:post)

data/lib/masamune/transform/define_table.psql.erb

@@ -22,22 +22,22 @@
 
 <%
   files ||= []
-  with_index = locals.fetch(:with_index, true)
-  with_foreign_key = locals.fetch(:with_foreign_key, true)
-  with_unique_constraint = locals.fetch(:with_unique_constraint, true)
 %>
 
 <%- target.children.each do |child| -%>
 <%= render 'define_table.psql.erb', target: child, **locals.except(:target, :files)  %>
 <%- end -%>
 
+<%- if helper.define_types? %>
 <%- target.enum_columns.each do |_, column| -%>
 DO $$ BEGIN
 IF NOT EXISTS (SELECT 1 FROM pg_type t WHERE LOWER(t.typname) = LOWER('<%= column.sql_type %>')) THEN
 CREATE TYPE <%= column.sql_type %> AS ENUM (<%= column.values.map { |value| "'#{value}'" }.join(', ') %>);
 END IF; END $$;
 <%- end -%>
+<%- end -%>
 
+<%- if helper.define_sequences? %>
 <%- target.sequence_columns.each do |_, column| -%>
 DO $$ BEGIN
 IF NOT EXISTS (SELECT 1 FROM pg_class c WHERE c.relname = '<%= column.sequence_id %>') THEN
@@ -45,7 +45,9 @@ CREATE SEQUENCE <%= column.sequence_id %>;
 ALTER SEQUENCE <%= column.sequence_id %> RESTART <%= column.sequence_offset %>;
 END IF; END $$;
 <%- end -%>
+<%- end -%>
 
+<%- if helper.define_tables? %>
 <%- if target.temporary? -%>
 CREATE TEMPORARY TABLE IF NOT EXISTS <%= target.name %>
 <%- else -%>
@@ -56,25 +58,29 @@ CREATE TABLE IF NOT EXISTS <%= target.name %>
   <%= column.as_psql %><%= ',' unless last %>
   <%- end -%>
 );
+<%- end -%>
 
-<%- unless target.temporary? || target.primary_keys.empty? -%>
+<%- if helper.define_primary_keys? %>
 DO $$ BEGIN
 IF NOT EXISTS (SELECT 1 FROM pg_class c WHERE c.relname = '<%= target.name %>_pkey') THEN
 ALTER TABLE <%= target.name %> ADD PRIMARY KEY (<%= target.primary_keys.map(&:name).join(', ') %>);
 END IF; END $$;
 <%- end -%>
 
-<%- if with_foreign_key && !target.delay_constraints? -%>
+<%- if helper.define_foreign_keys? -%>
 <%= render 'define_foreign_key.psql.erb', target: target %>
 <%- end -%>
 
+<%- if helper.define_sequences? -%>
 <%- target.sequence_columns.each do |_, column| -%>
 DO $$ BEGIN
 IF NOT EXISTS (SELECT 1 WHERE sequence_owner('<%= column.sequence_id %>') = '<%= column.qualified_name %>') THEN
 ALTER SEQUENCE <%= column.sequence_id %> OWNED BY <%= column.qualified_name %>;
 END IF; END $$;
 <%- end -%>
+<%- end -%>
 
+<%- if helper.load_files? -%>
 <%- files.each do |file| -%>
 <%-
   copy_options = []
@@ -84,25 +90,33 @@ END IF; END $$;
 -%>
 COPY <%= target.name %> FROM '<%= file %>' WITH (<%= copy_options.join(", ") %>);
 <%- end -%>
+<%- end -%>
 
-<%- if with_unique_constraint && !target.delay_constraints? -%>
+<%- if helper.define_inheritance? -%>
+<%= render 'define_inheritance.psql.erb', target: target %>
+<%- end -%>
+
+<%- if helper.define_unique_constraints? -%>
 <%= render 'define_unique.psql.erb', target: target %>
 <%- end -%>
 
-<%- if with_index && !target.delay_index? -%>
+<%- if helper.define_indexes? -%>
 <%= render 'define_index.psql.erb', target: target %>
 <%- end -%>
 
+<%- if helper.insert_rows? -%>
 <% target.insert_rows.each do |row| %>
 INSERT INTO <%= target.name %> (<%= row.insert_columns.join(', ') %>)
 SELECT <%= row.insert_values.join(', ') %>
 WHERE NOT EXISTS (SELECT 1 FROM <%= target.name %> WHERE <%= row.insert_constraints.join(' AND ') %>);
 <%- end -%>
+<%- end -%>
 
-<%- if files.any? || target.insert_rows.any? -%>
+<%- if helper.perform_analyze? -%>
 ANALYZE <%= target.name %>;
 <%- end -%>
 
+<%- if helper.define_functions? -%>
 <% target.aliased_rows.each do |row| %>
 <%- row.natural_keys.each do |column| -%>
 CREATE OR REPLACE FUNCTION <%= row.name(column) %>
@@ -118,3 +132,4 @@ RETURNS <%= target.surrogate_key.sql_type %> IMMUTABLE AS $$
 $$ LANGUAGE SQL;
 
 <%- end -%>
+<%- end -%>

data/lib/masamune/transform/define_table.rb

@@ -24,23 +24,114 @@ module Masamune::Transform
   module DefineTable
     extend ActiveSupport::Concern
 
-    def define_table(target, files = [], options = {})
+    def define_table(target, files = [], section = nil)
       return if target.implicit
-      Operator.new(__method__, target: target, files: Masamune::Schema::Map.convert_files(files), **options.slice(:with_index, :with_foreign_key, :with_unique_constraint), presenters: { postgres: Postgres, hive: Hive }).tap do |operator|
+      Operator.new(__method__, target: target, files: Masamune::Schema::Map.convert_files(files), section: section, helper: Helper, presenters: { postgres: Postgres, hive: Hive }).tap do |operator|
         logger.debug("#{target.id}\n" + operator.to_s) if target.debug
       end
     end
 
+    class Helper < SimpleDelegator
+      def files
+        locals[:files]
+      end
+
+      def section
+        locals[:section] || :all
+      end
+
+      def define_types?
+        !post_section?
+      end
+
+      def define_tables?
+        !post_section?
+      end
+
+      def define_functions?
+        !post_section?
+      end
+
+      def define_sequences?
+        !post_section?
+      end
+
+      def define_primary_keys?
+        !pre_section? && !(target.temporary? || target.primary_keys.empty?)
+      end
+
+      def define_inheritance?
+        return false unless target.inherited?
+        return false if pre_section?
+        return true if post_section?
+        !target.delay_indexes?
+      end
+
+      def define_indexes?
+        return false if pre_section?
+        return true if post_section?
+        !target.delay_indexes?
+      end
+
+      def define_foreign_keys?
+        return false if pre_section?
+        return true if post_section?
+        !target.delay_foreign_keys?
+      end
+
+      def define_unique_constraints?
+        return false if pre_section?
+        return true if post_section?
+        !target.delay_unique_constraints?
+      end
+
+      def insert_rows?
+        !post_section?
+      end
+
+      def load_files?
+        all_section?
+      end
+
+      def perform_analyze?
+        return false if pre_section?
+        return true if post_section?
+        files.any? || target.insert_rows.any?
+      end
+
+      private
+
+      def all_section?
+        section == :all
+      end
+
+      def pre_section?
+        section == :pre
+      end
+
+      def post_section?
+        section == :post
+      end
+    end
+
     class Postgres < SimpleDelegator
       def children
         super.map { |child| self.class.new(child) }
       end
 
-      def delay_index?
+      def inherited?
+        type == :fact && inheritance_constraints
+      end
+
+      def delay_indexes?
+        type == :fact
+      end
+
+      def delay_foreign_keys?
         type == :fact
       end
 
-      def delay_constraints?
+      def delay_unique_constraints?
         type == :fact
       end
     end