masamune 0.11.0

data/lib/masamune/schema/row.rb

@@ -0,0 +1,133 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Schema
+  class Row
+    DEFAULT_ATTRIBUTES =
+    {
+      id:       nil,
+      values:   {},
+      default:  false,
+      strict:   true,
+      parent:   nil,
+      debug:    false
+    }
+
+    DEFAULT_ATTRIBUTES.keys.each do |attr|
+      attr_accessor attr
+    end
+
+    def initialize(opts = {})
+      opts.symbolize_keys!
+      DEFAULT_ATTRIBUTES.merge(opts).each do |name, value|
+        public_send("#{name}=", value)
+      end
+      self.id ||= :default if default
+    end
+
+    def id=(id)
+      @id = id.to_sym if id
+    end
+
+    def values=(values)
+      @values = values.symbolize_keys
+    end
+
+    def parent=(parent)
+      @parent = parent
+      normalize_values! if @parent
+    end
+
+    def name(column = nil)
+      return unless @id
+      if column
+        "#{@id}_#{column.name}()"
+      else
+        "#{@id}_#{parent.name}_#{parent.surrogate_key.name}()"
+      end
+    end
+
+    def natural_keys
+      parent.natural_keys.select do |column|
+        values.keys.include?(column.name) && !column.sql_function?(values[column.name])
+      end
+    end
+
+    def insert_constraints
+      values.map { |key, value| "#{key} = #{parent.columns[key].sql_value(value)}" }.compact
+    end
+
+    def insert_columns
+      values.keys
+    end
+
+    def insert_values
+      values.map { |key, value| parent.columns[key].sql_value(value) }
+    end
+
+    def to_hash
+      values.with_indifferent_access
+    end
+
+    def headers
+      values.keys
+    end
+
+    def serialize
+      [].tap do |result|
+        values.each do |key, value|
+          result << @columns[key].csv_value(value)
+        end
+      end
+    end
+
+    def sql_value(column)
+      column.sql_value(values[column.name])
+    end
+
+    def missing_required_columns
+      Set.new.tap do |missing|
+        values.select do |key, value|
+          column = @columns[key]
+          missing << column if column.required_value? && value.nil?
+        end
+      end
+    end
+
+    private
+
+    def normalize_values!
+      result = {}
+      @columns = {}
+      values.each do |key, value|
+        next unless key
+        if column = parent.dereference_column_name(key)
+          @columns[column.name] = column
+          result[column.name] = column.ruby_value(value)
+        elsif strict
+          raise ArgumentError, "#{@values} contains undefined columns #{key}"
+        end
+      end
+      @values = result
+    end
+  end
+end

data/lib/masamune/schema/store.rb

@@ -0,0 +1,115 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'active_support/core_ext/hash'
+
+module Masamune::Schema
+  class Store
+    include Masamune::HasEnvironment
+
+    SUPPORTED_ATTRIBUTES = %(table dimension fact file event)
+
+    DEFAULT_ATTRIBUTES =
+    {
+      type:            nil,
+      format:          ->(store) { store.type == :postgres ? :csv : :tsv },
+      json_encoding:   ->(store) { store.type == :postgres ? :quoted : :raw },
+      headers:         ->(store) { store.type == :postgres ? true : false },
+      debug:           false
+    }
+
+    DEFAULT_ATTRIBUTES.keys.each do |attr|
+      attr_accessor attr
+    end
+
+    attr_accessor :tables
+    attr_accessor :dimensions
+    attr_accessor :facts
+    attr_accessor :files
+    attr_accessor :events
+    attr_accessor :references
+
+    class << self
+      def types
+        [:postgres, :hive, :files]
+      end
+    end
+
+    def initialize(environment, opts = {})
+      self.environment = environment
+      opts.symbolize_keys!
+      raise ArgumentError, 'required parameter type: missing' unless opts.key?(:type)
+      raise ArgumentError, "unknown type: '#{opts[:type]}'" unless self.class.types.include?(opts[:type])
+      DEFAULT_ATTRIBUTES.merge(opts).each do |name, value|
+        public_send("#{name}=", value.respond_to?(:call) ? value.call(self) : value)
+      end
+
+      @tables     = {}.with_indifferent_access
+      @dimensions = {}.with_indifferent_access
+      @facts      = {}.with_indifferent_access
+      @files      = {}.with_indifferent_access
+      @events     = {}.with_indifferent_access
+      @references = {}.with_indifferent_access
+      @extra      = []
+    end
+
+    def method_missing(method, *args, &block)
+      if type == :files
+        files[method]
+      else
+        *attribute_name, attribute_type = method.to_s.split('_')
+        raise ArgumentError, "unknown attribute type '#{attribute_type}'" unless SUPPORTED_ATTRIBUTES.include?(attribute_type)
+        self.send(attribute_type.pluralize)[attribute_name.join('_')]
+      end
+    end
+
+    def dereference_column(id, options = {})
+      column_id, reference_id = id.to_s.split(/\./).reverse
+      column_options = options.dup
+      column_options.merge!(id: column_id)
+
+      if reference = references[reference_id]
+        column_options.merge!(reference: reference)
+      else
+        raise ArgumentError, "dimension #{reference_id} not defined"
+      end if reference_id
+
+      Masamune::Schema::Column.new(column_options)
+    end
+
+    def extra(order = nil)
+      return @extra unless order
+      result = Set.new
+      @extra.each do |file|
+        filename = File.basename(file)
+        if filename =~ /\A\d+_/
+          number = filename.split('_').first.to_i
+          result << file if number <= 0 && order == :pre
+          result << file if number > 0 && order == :post
+        else
+          result << file if order == :pre
+        end
+      end
+      result.to_a
+    end
+  end
+end

data/lib/masamune/schema/table.rb

@@ -0,0 +1,308 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Schema
+  class Table
+    include Masamune::LastElement
+
+    attr_reader :children
+
+    DEFAULT_ATTRIBUTES =
+    {
+      id:              nil,
+      name:            nil,
+      type:            :table,
+      store:           nil,
+      parent:          nil,
+      suffix:          nil,
+      implicit:        false,
+      references:      {},
+      columns:         {},
+      rows:            [],
+      inherit:         false,
+      debug:           false
+    }
+
+    DEFAULT_ATTRIBUTES.keys.each do |attr|
+      attr_accessor attr
+    end
+
+    def initialize(opts = {})
+      opts.symbolize_keys!
+      raise ArgumentError, 'required parameter id: missing' unless opts.key?(:id)
+      DEFAULT_ATTRIBUTES.merge(opts).each do |name, value|
+        public_send("#{name}=", value)
+      end
+      @children = Set.new
+    end
+
+    def id=(id)
+      @id = id.to_sym
+    end
+
+    def references=(instance)
+      @references = {}
+      references = (instance.is_a?(Hash) ? instance.values : instance).compact
+      references.each do |reference|
+        raise ArgumentError, "table #{name} contains invalid table references" unless reference.is_a?(TableReference)
+        @references[reference.id] = reference
+      end
+    end
+
+    def columns=(instance)
+      @columns = {}
+      columns = (instance.is_a?(Hash) ? instance.values : instance).compact
+      raise ArgumentError, "table #{name} contains reserved columns" if columns.any? { |column| reserved_column_ids.include?(column.id) }
+
+      initialize_surrogate_key_column! unless columns.any? { |column| column.surrogate_key }
+      initialize_reference_columns!  unless columns.any? { |column| column.reference }
+      columns.each do |column|
+        raise ArgumentError, "table #{name} contains invalid columns" unless column.is_a?(Column)
+        @columns[column.name] = column.dup
+        @columns[column.name].parent = self
+      end
+    end
+
+    def rows=(rows)
+      @rows = []
+      rows.each do |row|
+        @rows << row.dup
+        @rows.last.parent = self
+      end
+    end
+
+    def name
+      @name || [id, suffix].compact.join('_')
+    end
+
+    def suffix
+      ((parent ? parent.suffix.split('_') : []) + [type.to_s, @suffix]).compact.uniq.join('_')
+    end
+
+    def temporary?
+      type == :stage
+    end
+
+    def surrogate_key
+      columns.values.detect { |column| column.surrogate_key }
+    end
+
+    def natural_keys
+      columns.values.select { |column| column.natural_key }
+    end
+
+    def defined_columns
+      columns.values
+    end
+    method_with_last_element :defined_columns
+
+    def unique_constraints
+      return [] if temporary?
+      unique_constraints_map.map do |_, column_names|
+        [column_names, short_md5(column_names)]
+      end
+    end
+
+    # TODO: Add optional USING
+    # TODO: Default to GIN for array columns
+    def index_columns
+      index_column_map.map do |_, column_names|
+        [column_names, reverse_unique_constraints_map.key?(column_names.sort), short_md5(column_names)]
+      end
+    end
+
+    def unique_columns
+      return {} if temporary?
+      columns.select { |_, column| column.unique }
+    end
+
+    def enum_columns
+      return {} if temporary?
+      columns.select { |_, column| column.type == :enum }
+    end
+
+    def sequence_columns
+      return {} if temporary?
+      columns.select { |_, column| column.reference.nil? && column.type == :sequence }
+    end
+
+    def reference_columns
+      columns.values.select { | column| column.reference }
+    end
+
+    def foreign_key_columns
+      columns.values.select { | column| column.reference && column.reference.foreign_key }
+    end
+
+    def insert_rows
+      rows.select { |row| row.insert_values.any? }
+    end
+
+    def aliased_rows
+      rows.select { |row| row.name }
+    end
+
+    def insert_references
+      references.select { |_, reference| reference.insert }
+    end
+
+    def reserved_columns
+      columns.select { |_, column| reserved_column_ids.include?(column.id) }
+    end
+
+    def unreserved_columns
+      columns.reject { |_, column| reserved_column_ids.include?(column.id) }
+    end
+
+    def stage_table(options = {})
+      selected = options[:columns] if options[:columns]
+      selected ||= options[:target].columns.values.map(&:compact_name) if options[:target]
+      selected ||= []
+      stage_id = [id, options[:suffix]].compact.join('_')
+      parent = options[:table] ? options[:table] : self
+      type = options[:type] ? options[:type] : :stage
+      @stage_tables ||= {}
+      @stage_tables[options] ||= parent.class.new id: stage_id, type: type, store: store, columns: stage_table_columns(parent, selected, options.fetch(:inherit, true)), references: stage_table_references(parent, selected), parent: parent, inherit: options.fetch(:inherit, true)
+    end
+
+    def shared_columns(other)
+      Hash.new { |h,k| h[k] = [] }.tap do |shared|
+        columns.each do |_, column|
+          other.columns.each do |_, other_column|
+            shared[column] << other_column if column.references?(other_column)
+          end
+        end
+      end
+    end
+
+    def dereference_column_name(name)
+      reference_name, column_name = Column::dereference_column_name(name)
+      if reference = references[reference_name]
+        if column = reference.columns[column_name]
+          dereference_column(column.dup, reference)
+        end
+      elsif column = columns[column_name]
+        column
+      end
+    end
+
+    def dereference_column(column, reference)
+      column.surrogate_key = false
+      column.reference = reference
+      column
+    end
+
+    def reserved_column_ids
+      inherit ? parent.reserved_column_ids : []
+    end
+
+    private
+
+    def stage_table_columns(parent, selected = [], inherit = true)
+      selected = columns.keys if selected.empty?
+      {}.tap do |result|
+        selected.each do |name|
+          column = dereference_column_name(name)
+          next unless column
+          next if inherit && parent.reserved_column_ids.include?(column.id)
+          if column.parent == self
+            next if column.surrogate_key
+            result[name] = column
+          else
+            result[name] = column
+          end
+        end
+      end
+    end
+
+    def stage_table_references(parent, selected = [])
+      selected = references.keys if selected.empty?
+      {}.tap do |result|
+        selected.each do |name|
+          column = dereference_column_name(name)
+          next unless column
+          next if column.parent == self
+          result[name] = column.reference
+        end
+      end
+    end
+
+    def initialize_surrogate_key_column!
+      case type
+      when :table
+        initialize_column! id: 'id', type: :integer, surrogate_key: true
+      end
+    end
+
+    def initialize_reference_columns!
+      references.map do |_, reference|
+        if reference.denormalize
+          reference.unreserved_columns.each do |_, column|
+            next if column.surrogate_key
+            next if column.ignore
+            initialize_column! id: column.id, type: column.type, reference: reference, default: reference.default, index: true, null: reference.null, natural_key: reference.natural_key
+          end
+        elsif reference.foreign_key
+          # FIXME column.reference should point to reference.surrogate_key, only allow column references to Columns
+          initialize_column! id: reference.foreign_key_name, type: reference.foreign_key_type, reference: reference, default: reference.default, index: true, null: reference.null, natural_key: reference.natural_key
+        end
+      end
+    end
+
+    def initialize_column!(options = {})
+      column = Masamune::Schema::Column.new(options.merge(parent: self))
+      @columns[column.name.to_sym] = column
+    end
+
+    def index_column_map
+      @index_column_map ||= begin
+        map = Hash.new { |h,k| h[k] = [] }
+        columns.each do |_, column|
+          column.index.each do |index|
+            map[index] << column.name
+          end
+        end
+        Hash[map.sort_by { |k, v| v.length }]
+      end
+    end
+
+    def unique_constraints_map
+      @unique_constraints_map ||= begin
+        map = Hash.new { |h,k| h[k] = [] }
+        columns.each do |_, column|
+          column.unique.each do |unique|
+            map[unique] << column.name
+          end
+        end
+        Hash[map.sort_by { |k, v| v.length }]
+      end
+    end
+
+    def reverse_unique_constraints_map
+      @reverse_unique_constraints_map ||= Hash[unique_constraints_map.to_a.map { |k,v| [v.sort, k] }]
+    end
+
+    def short_md5(*a)
+      Digest::MD5.hexdigest(a.join('_'))[0..6]
+    end
+  end
+end