masamune 0.12.3 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 18fd59e8bd668c7efc320ad374ef1bc96c384184
-  data.tar.gz: f12fa656c4136942e1d4a5a917788c2d692b7e8f
+  metadata.gz: 77bd1d9e8351f259a7d946ac7b2abd11b4c14408
+  data.tar.gz: deb47f8efaadd8b90b5825f79970e46833bbb030
 SHA512:
-  metadata.gz: f83bd5adbc40df655a01e4a390d6435fd5bc6e26eeae21ea13898a31931ee6b878540988a0c2de02b082f7e5f86e21e53152e9f704df396f1cda7e8dba1a3cd2
-  data.tar.gz: af6239a670e87aa6757b6dea3c96c467b4243b7b86cbd3ce35ae46dd5e7ff1362751ede5cb1c7a9e686716ef410e5e976fe2ebe65e5c948717de24713ed44445
+  metadata.gz: 784416ce6169e2c3f2a9a14465580417ecfc6bdd73e238643022f608a9a04ede66d0b670dfce06e0956b14d3d96a8edbcbe0018ee76bdd0e1ccd4c2bb9f4b37a
+  data.tar.gz: 6b1cb22550d392cb6bf6cf36258d21b28e7c323dd6bab5616b4066380e315bcb289c725d52a36f65cd182293c1a28d68f31e928a642d5429ce1f024945f2d81a

data/bin/masamune-dump

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+$: << File.expand_path('../../lib/', __FILE__)
+require 'masamune/tasks/dump_thor'
+Masamune::Tasks::DumpThor.start(ARGV)

data/lib/masamune/schema/column.rb

@@ -85,7 +85,6 @@ module Masamune::Schema
 
     def index=(value)
       @index ||= Set.new
-      @index.clear
       @index +=
       case value
       when true
@@ -101,14 +100,8 @@ module Masamune::Schema
       end
     end
 
-    def unique
-      self.unique = 'natural' if natural_key
-      @unique
-    end
-
     def unique=(value)
       @unique ||= Set.new
-      @unique.clear
       @unique +=
       case value
       when true
@@ -125,7 +118,7 @@ module Masamune::Schema
     end
 
     def foreign_key_name
-      "#{reference.name}.#{@id}".to_sym if reference
+      "#{[reference.label, reference.name].compact.join('_')}.#{@id}".to_sym if reference
     end
 
     def compact_name
@@ -363,7 +356,7 @@ module Masamune::Schema
     end
 
     def as_psql
-      [name, sql_type(surrogate_key), *sql_constraints, reference_constraint, sql_default].compact.join(' ')
+      [name, sql_type(surrogate_key), *sql_constraints, sql_default].compact.join(' ')
     end
 
     def as_hql
@@ -378,16 +371,6 @@ module Masamune::Schema
       end
     end
 
-    # TODO: Add ELEMENT REFERENCES
-    def reference_constraint
-      return if parent && parent.temporary?
-      return if degenerate?
-      return if array_value?
-      if reference && reference.surrogate_key.type == type
-        "REFERENCES #{reference.name}(#{reference.surrogate_key.name})"
-      end
-    end
-
     class << self
       def dereference_column_name(name)
         return unless name
@@ -477,7 +460,6 @@ module Masamune::Schema
     def sql_constraints
       [].tap do |constraints|
         constraints << 'NOT NULL' unless null || surrogate_key || !strict || parent.temporary? || degenerate?
-        constraints << "PRIMARY KEY" if surrogate_key
       end
     end
 

data/lib/masamune/schema/dimension.rb

@@ -91,27 +91,28 @@ module Masamune::Schema
     end
 
     def initialize_dimension_columns!
+      # TODO assign index for load_fact
       case type
       when :one, :date
         initialize_column! id: 'last_modified_at', type: :timestamp, default: 'NOW()'
       when :two
-        initialize_column! id: 'start_at', type: :timestamp, default: 'TO_TIMESTAMP(0)', index: true, unique: 'natural'
-        initialize_column! id: 'end_at', type: :timestamp, null: true, index: true
-        initialize_column! id: 'version', type: :integer, default: 1, null: true, index: true
+        initialize_column! id: 'start_at', type: :timestamp, default: 'TO_TIMESTAMP(0)', index: [:start_at, :natural], unique: :natural
+        initialize_column! id: 'end_at', type: :timestamp, null: true, index: :end_at
+        initialize_column! id: 'version', type: :integer, default: 1, null: true
         initialize_column! id: 'last_modified_at', type: :timestamp, default: 'NOW()'
       when :four
         children << ledger_table
         # FIXME derive type from from parent
-        initialize_column! id: 'parent_id', type: :integer, null: true, reference: ledger_table
-        initialize_column! id: 'record_id', type: :integer, null: true, reference: ledger_table
-        initialize_column! id: 'start_at', type: :timestamp, default: 'TO_TIMESTAMP(0)', index: true, unique: 'natural'
-        initialize_column! id: 'end_at', type: :timestamp, null: true, index: true
-        initialize_column! id: 'version', type: :integer, default: 1, null: true, index: true
+        initialize_column! id: 'parent_id', type: :integer, null: true, reference: TableReference.new(ledger_table)
+        initialize_column! id: 'record_id', type: :integer, null: true, reference: TableReference.new(ledger_table)
+        initialize_column! id: 'start_at', type: :timestamp, default: 'TO_TIMESTAMP(0)', index: [:start_at, :natural], unique: :natural
+        initialize_column! id: 'end_at', type: :timestamp, null: true, index: :end_at
+        initialize_column! id: 'version', type: :integer, default: 1, null: true
         initialize_column! id: 'last_modified_at', type: :timestamp, default: 'NOW()'
       when :ledger
-        initialize_column! id: 'source_kind', type: :string, unique: 'natural'
-        initialize_column! id: 'source_uuid', type: :string, unique: 'natural'
-        initialize_column! id: 'start_at', type: :timestamp, index: true, unique: 'natural'
+        initialize_column! id: 'source_kind', type: :string, index: :natural, unique: :natural
+        initialize_column! id: 'source_uuid', type: :string, index: :natural, unique: :natural
+        initialize_column! id: 'start_at', type: :timestamp, index: :natural, unique: :natural
         initialize_column! id: 'last_modified_at', type: :timestamp, default: 'NOW()'
         initialize_column! id: 'delta', type: :integer
       when :stage

data/lib/masamune/schema/fact.rb

@@ -34,8 +34,13 @@ module Masamune::Schema
       @partition = opts.delete(:partition)
       super opts.reverse_merge(type: :fact)
       initialize_fact_columns!
-      foreign_key_columns.each do |column|
+      reference_columns.each do |column|
+        column.index.clear
         column.index << column.name
+        if type == :stage
+          column.index << "#{column.name}_time_key"
+          time_key.index << "#{column.name}_time_key"
+        end
       end
       time_key.index << time_key.name
     end
@@ -59,6 +64,10 @@ module Masamune::Schema
       columns.select { |_, column| column && column.reference && column.reference.type == :date }.values.first
     end
 
+    def primary_keys
+      []
+    end
+
     def time_key
       columns.values.detect { |column| column.id == :time_key }
     end

data/lib/masamune/schema/map.rb

@@ -112,7 +112,7 @@ module Masamune::Schema
         @csv ||= CSV.new(@io, options.merge(headers: row.headers, write_headers: write_headers))
         if row.missing_required_columns.any?
           missing_required_column_names = row.missing_required_columns.map(&:name)
-          @store.logger.warn("row '#{row.to_hash}' is missing required columns '#{missing_required_column_names.join(',')}', skipping")
+          @store.logger.warn("row '#{row.to_hash}' is missing required columns '#{missing_required_column_names.join(', ')}', skipping")
         else
           @csv << row.serialize if append?(row.serialize)
         end
@@ -258,8 +258,8 @@ module Masamune::Schema
       Array.wrap(function.call(input)).each do |output|
         yield output
       end
-    rescue
-      @store.logger.warn("failed to process '#{input}' for #{target.name}, skipping")
+    rescue => e
+      @store.logger.warn("failed to process row for #{target.name}, skipping: #{e.message}")
     end
   end
 end

data/lib/masamune/schema/row.rb

@@ -60,7 +60,7 @@ module Masamune::Schema
     def name(column = nil)
       return unless @id
       if column
-        "#{@id}_#{column.name}()"
+        "#{@id}_#{parent.name}_#{column.name}()"
       else
         "#{@id}_#{parent.name}_#{parent.surrogate_key.name}()"
       end

data/lib/masamune/schema/table.rb

@@ -75,11 +75,9 @@ module Masamune::Schema
       raise ArgumentError, "table #{name} contains reserved columns" if columns.any? { |column| reserved_column_ids.include?(column.id) }
 
       initialize_surrogate_key_column! unless columns.any? { |column| column.surrogate_key }
-      initialize_reference_columns!  unless columns.any? { |column| column.reference }
+      initialize_reference_columns! unless columns.any? { |column| column.reference }
       columns.each do |column|
-        raise ArgumentError, "table #{name} contains invalid columns" unless column.is_a?(Column)
-        @columns[column.name] = column.dup
-        @columns[column.name].parent = self
+        initialize_column!(column)
       end
     end
 
@@ -107,6 +105,10 @@ module Masamune::Schema
       columns.values.detect { |column| column.surrogate_key }
     end
 
+    def primary_keys
+      [*auto_surrogate_keys, surrogate_key].compact
+    end
+
     def natural_keys
       columns.values.select { |column| column.natural_key }
     end
@@ -119,16 +121,14 @@ module Masamune::Schema
     def unique_constraints
       return [] if temporary?
       unique_constraints_map.map do |_, column_names|
-        [column_names, short_md5(column_names)]
+        [column_names, short_md5(column_names.to_a)]
       end.uniq
     end
 
-    # TODO: Add optional USING
-    # TODO: Default to GIN for array columns
     def index_columns
       index_column_map.map do |_, column_names|
         unique_index = reverse_unique_constraints_map.key?(column_names.sort)
-        [column_names, unique_index, short_md5(column_names)]
+        [column_names, unique_index, short_md5(column_names.to_a)]
       end.uniq
     end
 
@@ -152,7 +152,7 @@ module Masamune::Schema
     end
 
     def foreign_key_columns
-      columns.values.select { | column| column.reference && column.reference.foreign_key }
+      columns.values.select { | column| !column.degenerate? && column.reference && column.reference.foreign_key }
     end
 
     def partitions
@@ -180,22 +180,28 @@ module Masamune::Schema
     end
 
     def denormalized_columns
+      return to_enum(__method__).to_a.flatten.compact unless block_given?
       columns.map do |_, column|
         next if column.surrogate_key || column.ignore
         if column.reference
-          column.reference.natural_keys.any? ? column.reference.natural_keys : column.reference.denormalized_columns
+          (column.reference.natural_keys.any? ? column.reference.natural_keys : column.reference.denormalized_columns).each do |join_column|
+            yield [column.reference, join_column]
+          end
         else
-          column
+          yield [nil, column]
         end
-      end.flatten.compact
+      end
     end
 
     def denormalized_column_names
-      denormalized_columns.map do |column|
+      return to_enum(__method__).to_a unless block_given?
+      denormalized_columns do |reference, column|
         if column.parent == self
-          column.name.to_s
+          yield column.name.to_s
+        elsif reference
+          yield [reference.id, column.name].compact.join('.')
         else
-          [column.parent.id, column.name].join('.')
+          yield [column.parent.id, column.name].compact.join('.')
         end
       end
     end
@@ -247,6 +253,24 @@ module Masamune::Schema
       Integer('0x' + Digest::MD5.hexdigest(name)) % (1 << 63)
     end
 
+    def auto_surrogate_keys
+      columns.values.select { |column| column.reference && column.reference.surrogate_key.auto }.uniq.compact
+    end
+
+    def foreign_key_constraints
+      return [] if temporary?
+      foreign_key_columns.map do |column|
+        if column.reference.auto_surrogate_keys == auto_surrogate_keys
+          column_names = [*column.reference.auto_surrogate_keys.map(&:name), column.name].compact
+          reference_column_names = [*column.reference.auto_surrogate_keys.map(&:name), column.reference.surrogate_key.name].compact
+        else
+          column_names = [column.name]
+          reference_column_names = [column.reference.surrogate_key.name]
+        end
+        [short_md5(column_names), column_names, column.reference.name, reference_column_names]
+      end.compact
+    end
+
     private
 
     def stage_table_columns(parent, selected = [], inherit = true)
@@ -291,18 +315,22 @@ module Masamune::Schema
           reference.unreserved_columns.each do |_, column|
             next if column.surrogate_key
             next if column.ignore
-            initialize_column! id: column.id, type: column.type, reference: reference, default: reference.default, index: true, null: reference.null, natural_key: reference.natural_key
+            initialize_column! id: column.id, type: column.type, reference: reference, default: reference.default, null: reference.null, natural_key: reference.natural_key
           end
         elsif reference.foreign_key
           # FIXME column.reference should point to reference.surrogate_key, only allow column references to Columns
-          initialize_column! id: reference.foreign_key_name, type: reference.foreign_key_type, reference: reference, default: reference.default, index: true, null: reference.null, natural_key: reference.natural_key
+          initialize_column! id: reference.foreign_key_name, type: reference.foreign_key_type, reference: reference, default: reference.default, null: reference.null, natural_key: reference.natural_key
         end
       end
     end
 
-    def initialize_column!(options = {})
-      column = Masamune::Schema::Column.new(options.merge(parent: self))
-      @columns[column.name.to_sym] = column
+    def initialize_column!(column_or_options)
+      column = column_or_options.is_a?(Column) ? column_or_options.dup : Column.new(column_or_options.merge(parent: self))
+      column_key = column.name.to_sym
+      @columns[column_key] = column
+      @columns[column_key].parent = self
+      @columns[column_key].index += [column_key, :natural] if column.natural_key
+      @columns[column_key].unique << :natural if column.natural_key
     end
 
     def index_column_map
@@ -311,9 +339,10 @@ module Masamune::Schema
         columns.each do |_, column|
           column.index.each do |index|
             map[index] << column.name
+            map[index].uniq!
           end
         end
-        Hash[map.sort_by { |k, v| v.length }]
+        Hash[map.sort_by { |k, v| [v.length, k.to_s] }]
       end
     end
 
@@ -321,11 +350,14 @@ module Masamune::Schema
       @unique_constraints_map ||= begin
         map = Hash.new { |h,k| h[k] = [] }
         columns.each do |_, column|
+          next if column.auto_reference
           column.unique.each do |unique|
+            map[unique] += auto_surrogate_keys.map(&:name)
             map[unique] << column.name
+            map[unique].uniq!
           end
         end unless temporary?
-        Hash[map.sort_by { |k, v| v.length }]
+        Hash[map.sort_by { |k, v| [v.length, k.to_s] }]
       end
     end
 
@@ -334,7 +366,7 @@ module Masamune::Schema
     end
 
     def short_md5(*a)
-      Digest::MD5.hexdigest(a.join('_'))[0..6]
+      Digest::MD5.hexdigest(a.compact.sort.uniq.join('_'))[0..6]
     end
   end
 end

data/lib/masamune/schema/table_reference.rb

@@ -31,6 +31,7 @@ module Masamune::Schema
       natural_key:     false,
       denormalize:     false,
       multiple:        false,
+      through:         [],
       degenerate:      false
     }
 
@@ -72,5 +73,9 @@ module Masamune::Schema
         default_row.name(column)
       end
     end
+
+    def through=(columns)
+      @through = Array.wrap(columns)
+    end
   end
 end

data/lib/masamune/tasks/dump_thor.rb

@@ -0,0 +1,58 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'masamune'
+require 'thor'
+
+module Masamune::Tasks
+  class DumpThor < Thor
+    include Masamune::Thor
+    include Masamune::Transform::DefineSchema
+
+    # FIXME need to add an unnecessary namespace until this issue is fixed:
+    # https://github.com/wycats/thor/pull/247
+    namespace :dump
+    skip_lock!
+
+    desc 'dump', 'Dump schema'
+    method_option :type, :enum => ['psql', 'hql'], :desc => 'Schema type', :default => 'psql'
+    method_option :with_index, :type => :boolean, :desc => 'Dump schema with indexes', :default => true
+    method_option :with_foreign_key, :type => :boolean, :desc => 'Dump schema with foreign key constraints', :default => true
+    method_option :with_unique_constraint, :type => :boolean, :desc => 'Dump schema with uniqueness constraints', :default => true
+    def dump_exec
+      print_catalog
+      exit
+    end
+    default_task :dump_exec
+
+    private
+
+    def print_catalog
+      case options[:type]
+      when 'psql'
+        puts define_schema(catalog, :postgres, options.slice(:with_index, :with_foreign_key, :with_unique_constraint).to_h.symbolize_keys)
+      when 'hql'
+        puts define_schema(catalog, :hive)
+      end
+    end
+  end
+end

data/lib/masamune/tasks/shell_thor.rb

@@ -28,7 +28,6 @@ module Masamune::Tasks
   class ShellThor < Thor
     include Masamune::Thor
     include Masamune::Actions::DataFlow
-    include Masamune::Transform::DefineSchema
 
     # FIXME need to add an unnecessary namespace until this issue is fixed:
     # https://github.com/wycats/thor/pull/247
@@ -36,29 +35,11 @@ module Masamune::Tasks
     skip_lock!
 
     desc 'shell', 'Launch an interactive shell'
-    method_option :dump, :type => :boolean, :desc => 'Dump SQL schema', :default => false
-    method_option :type, :enum => ['psql', 'hql'], :desc => 'Schema type', :default => 'psql'
     method_option :prompt, :desc => 'Set shell prompt', :default => 'masamune'
     class_option :start, :aliases => '-a', :desc => 'Start time', default: '1 month ago'
     def shell_exec
-      if options[:dump]
-        print_catalog
-        exit
-      end
-
       Pry.start self, prompt: proc { options[:prompt] + '> ' }
     end
     default_task :shell_exec
-
-    private
-
-    def print_catalog
-      case options[:type]
-      when 'psql'
-        puts define_schema(catalog, :postgres)
-      when 'hql'
-        puts define_schema(catalog, :hive)
-      end
-    end
   end
 end