masamune 0.11.0

data/lib/masamune/schema/dimension.rb

@@ -0,0 +1,133 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Schema
+  class Dimension < Table
+    def initialize(opts = {})
+      super
+      initialize_dimension_columns!
+    end
+
+    def suffix
+      suffix = case type
+      when :mini
+        'type'
+      when :one, :two, :four, :date
+        'dimension'
+      else
+        type.to_s
+      end
+      parent ? [parent.suffix, suffix].compact.join('_') : suffix
+    end
+
+    def start_key
+      columns.values.detect { |column| column.id == :start_at }
+    end
+
+    def end_key
+      columns.values.detect { |column| column.id == :end_at }
+    end
+
+    def version_key
+      columns.values.detect { |column| column.id == :version }
+    end
+
+    def ledger_table
+      @ledger_table ||= self.class.new(id: id, type: :ledger, store: store, columns: ledger_table_columns, references: references.values, parent: self)
+    end
+
+    def reserved_column_ids
+      case type
+      when :one, :date
+        [:last_modified_at]
+      when :two
+        [:start_at, :end_at, :version, :last_modified_at]
+      when :four
+        [:parent_id, :record_id, :start_at, :end_at, :version, :last_modified_at]
+      when :ledger
+        [:source_kind, :source_uuid, :start_at, :last_modified_at, :delta]
+      else
+        super
+      end
+    end
+
+    private
+
+    def ledger_table_columns
+      columns.values.map do |column|
+        next if column.surrogate_key
+        next if reserved_column_ids.include?(column.id)
+
+        if column.type == :key_value
+          column_now, column_was = column.dup, column.dup
+          column_now.id, column_was.id = "#{column.id}_now", "#{column.id}_was"
+          column_now.strict, column_was.strict = false, false
+          [column_now, column_was]
+        else
+          column.dup.tap do |column_copy|
+            column_copy.strict = false unless column.surrogate_key || column.natural_key || (column.reference && column.reference.surrogate_key.auto)
+          end
+        end
+      end.flatten
+    end
+
+    def initialize_surrogate_key_column!
+      case type
+      when :mini, :one, :two, :four, :ledger, :date
+        initialize_column! id: 'id', type: :integer, surrogate_key: true
+      end
+    end
+
+    def initialize_dimension_columns!
+      case type
+      when :one, :date
+        initialize_column! id: 'last_modified_at', type: :timestamp, default: 'NOW()'
+      when :two
+        initialize_column! id: 'start_at', type: :timestamp, default: 'TO_TIMESTAMP(0)', index: true, unique: 'natural'
+        initialize_column! id: 'end_at', type: :timestamp, null: true, index: true
+        initialize_column! id: 'version', type: :integer, default: 1, null: true, index: true
+        initialize_column! id: 'last_modified_at', type: :timestamp, default: 'NOW()'
+      when :four
+        children << ledger_table
+        # FIXME derive type from from parent
+        initialize_column! id: 'parent_id', type: :integer, null: true, reference: ledger_table
+        initialize_column! id: 'record_id', type: :integer, null: true, reference: ledger_table
+        initialize_column! id: 'start_at', type: :timestamp, default: 'TO_TIMESTAMP(0)', index: true, unique: 'natural'
+        initialize_column! id: 'end_at', type: :timestamp, null: true, index: true
+        initialize_column! id: 'version', type: :integer, default: 1, null: true, index: true
+        initialize_column! id: 'last_modified_at', type: :timestamp, default: 'NOW()'
+      when :ledger
+        initialize_column! id: 'source_kind', type: :string, unique: 'natural'
+        initialize_column! id: 'source_uuid', type: :string, unique: 'natural'
+        initialize_column! id: 'start_at', type: :timestamp, index: true, unique: 'natural'
+        initialize_column! id: 'last_modified_at', type: :timestamp, default: 'NOW()'
+        initialize_column! id: 'delta', type: :integer
+      when :stage
+        if inherit
+          parent.reserved_columns.each do |_, column|
+            initialize_column! column.as_hash
+          end
+        end
+      end
+    end
+  end
+end

data/lib/masamune/schema/event.rb

@@ -0,0 +1,121 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Schema
+  class Event
+    class Attribute
+      attr_accessor :id
+      attr_accessor :type
+      attr_accessor :array
+      attr_accessor :immutable
+
+      def initialize(opts = {})
+        opts.symbolize_keys!
+        raise ArgumentError, 'required parameter id: missing' unless opts.key?(:id)
+        self.id = opts[:id].to_sym
+        self.type = opts.fetch(:type, :integer).to_sym
+        self.array = opts.fetch(:array, false)
+        self.immutable = opts.fetch(:immutable, false)
+      end
+
+      def as_columns(event, &block)
+        column_ids = immutable ? [id] : [:"#{id}_now", :"#{id}_was"]
+        column_ids.each do |id|
+          yield [id, Column.new(id: id, type: type, array: array, parent: event)]
+        end
+      end
+    end
+
+    DEFAULT_ATTRIBUTES =
+    {
+      id:              nil,
+      store:           nil,
+      attributes:      [],
+      debug:           false
+    }
+
+    DEFAULT_ATTRIBUTES.keys.each do |attr|
+      attr_accessor attr
+    end
+
+    def initialize(opts = {})
+      opts.symbolize_keys!
+      raise ArgumentError, 'required parameter id: missing' unless opts.key?(:id)
+      DEFAULT_ATTRIBUTES.merge(opts).each do |name, value|
+        public_send("#{name}=", value)
+      end
+    end
+
+    def name
+      @name || [id, suffix].compact.join('_')
+    end
+
+    def suffix
+      'event'
+    end
+
+    def attributes=(attributes)
+      @attributes = {}
+      attributes.each do |attribute|
+        @attributes[attribute.id] = attribute
+      end
+    end
+
+    def columns
+      @columns ||= {}.tap do |columns|
+        columns[:uuid] = Column.new id: :uuid, type: :uuid, parent: self
+        columns[:type] = Column.new id: :type, type: :string, parent: self
+        attributes.map do |_, attribute|
+          attribute.as_columns(self) do |id, column|
+            columns[id] = column
+          end
+        end
+        columns[:delta] = Column.new id: :delta, type: :integer, parent: self
+        columns[:created_at] = Column.new id: :created_at, type: :timestamp, parent: self
+      end
+    end
+
+    def reserved_column_ids
+      @reserved_column_ids ||= [:uuid, :type, :delta, :created_at]
+    end
+
+    def unreserved_columns
+      columns.reject { |_, column| reserved_column_ids.include?(column.id) }
+    end
+
+    def create_type
+      @create_type ||= "#{id}_create"
+    end
+
+    def update_type
+      @update_type ||= "#{id}_update"
+    end
+
+    def delete_type
+      @delete_type ||= "#{id}_delete"
+    end
+
+    def dereference_column_name(name)
+      columns[name.to_sym]
+    end
+  end
+end

data/lib/masamune/schema/fact.rb

@@ -0,0 +1,133 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Schema
+  class Fact < Table
+    SUPPORTED_GRAINS = [:transaction, :hourly, :daily, :monthly]
+
+    attr_accessor :grain
+    attr_accessor :partition
+    attr_accessor :range
+
+    def initialize(opts = {})
+      opts.symbolize_keys!
+      self.grain = opts.delete(:grain)
+      @partition = opts.delete(:partition)
+      super opts.reverse_merge(type: :fact)
+      initialize_fact_columns!
+      foreign_key_columns.each do |column|
+        column.index << column.name
+      end
+      time_key.index << time_key.name
+    end
+
+    def id
+      [@id, grain].compact.join('_').to_sym
+    end
+
+    def grain=(grain = nil)
+      return unless grain
+      raise ArgumentError, "unknown grain '#{grain}'" unless SUPPORTED_GRAINS.include?(grain.to_sym)
+      @grain = grain.to_sym
+    end
+
+    def suffix
+      inherited = super
+      [*inherited.split('_'), range.try(:suffix)].compact.uniq.join('_')
+    end
+
+    def date_column
+      columns.select { |_, column| column && column.reference && column.reference.type == :date }.values.first
+    end
+
+    def time_key
+      columns.values.detect { |column| column.id == :time_key }
+    end
+
+    def stage_table(*a)
+      super.tap do |stage|
+        stage.id    = @id
+        stage.store = store
+        stage.range = range
+        stage.grain = grain
+        stage.columns.each do |_, column|
+          column.unique = false
+        end
+      end
+    end
+
+    def partition_table(date)
+      partition_range = partition_rule.bind_date(date)
+      @partition_tables ||= {}
+      @partition_tables[partition_range] ||= self.class.new(id: @id, store: store, columns: partition_table_columns, parent: self, range: partition_range, grain: grain, inherit: true)
+    end
+
+    def partitions
+      columns.select { |_, column| column.partition }
+    end
+
+    def measures
+      columns.select { |_, column| column.measure }
+    end
+
+    def constraints
+      return unless range
+      "CHECK (time_key >= #{range.start_time.to_i} AND time_key < #{range.stop_time.to_i})"
+    end
+
+    def reserved_column_ids
+      case type
+      when :fact
+        [:time_key, :last_modified_at]
+      else
+        super
+      end
+    end
+
+    private
+
+    def initialize_surrogate_key_column!
+    end
+
+    def initialize_fact_columns!
+      case type
+      when :fact
+        initialize_column! id: 'time_key', type: :integer, index: true
+        initialize_column! id: 'last_modified_at', type: :timestamp, default: 'NOW()' unless store.type == :hive
+      when :stage
+        if inherit
+          parent.reserved_columns.each do |_, column|
+            initialize_column! column.as_hash
+          end
+        end
+      end
+    end
+
+    def partition_rule
+      @partition_rule ||= Masamune::DataPlan::Rule.new(nil, :tmp, :target, table: name, partition: @partition)
+    end
+
+    def partition_table_columns
+      unreserved_columns.map { |_, column| column.dup }
+    end
+  end
+end

data/lib/masamune/schema/map.rb

@@ -0,0 +1,265 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'csv'
+
+module Masamune::Schema
+  class Map
+    class JSONEncoder < SimpleDelegator
+      def initialize(io, store)
+        super io
+        @store = store
+      end
+
+      def gets(*a)
+        line = __getobj__.gets(*a)
+        return unless line
+        return line if skip?
+        encode(line, separator).join(separator)
+      end
+
+      private
+
+      def skip?
+        @store.json_encoding == :quoted
+      end
+
+      def encode(line, separator)
+        fields = []
+        buffer = ''
+        nested = false
+        line.strip.each_char do |char|
+          case char
+          when '{'
+            buffer << char
+            nested = true
+          when '}'
+            buffer << char
+            nested = false
+          when separator
+            if nested
+              buffer << char
+            else
+              fields << quote(buffer)
+              buffer = ''
+            end
+          else
+            buffer << char
+          end
+        end
+        fields << quote(buffer)
+        fields.compact
+      end
+
+      def quote(buffer)
+        return buffer if buffer =~ /\A".*"\z/
+        %Q{"#{buffer.gsub('"', '""')}"}
+      end
+
+      def separator
+        @separator ||= (@store.format == :tsv ? "\t" : ',')
+      end
+    end
+
+    class Buffer
+      extend Forwardable
+
+      def_delegators :@io, :flush, :path
+
+      def initialize(table, options = {})
+        @table    = table
+        @store    = table.store
+        @lines    = 0
+        @options  = options
+      end
+
+      def bind(io)
+        @io = io.set_encoding('binary', 'UTF-8', undef: :replace)
+        @csv = nil
+      end
+
+      def each(&block)
+        raise 'must call Buffer#bind first' unless @io
+        CSV.parse(JSONEncoder.new(@io, @store), options.merge(headers: @store.headers || @table.columns.keys)) do |data|
+          next if data.to_s =~ /\A#/
+          yield safe_row(data)
+        end
+      end
+
+      def append(data)
+        raise 'must call Buffer#bind first' unless @io
+        row = Masamune::Schema::Row.new(parent: @table, values: data.to_hash)
+        write_headers = @store.headers && @lines < 1
+        @csv ||= CSV.new(@io, options.merge(headers: row.headers, write_headers: write_headers))
+        if row.missing_required_columns.any?
+          missing_required_column_names = row.missing_required_columns.map(&:name)
+          @store.logger.warn("row '#{row.to_hash}' is missing required columns '#{missing_required_column_names.join(',')}', skipping")
+        else
+          @csv << row.serialize if append?(row.serialize)
+        end
+        @lines += 1
+      end
+
+      private
+
+      def options
+        {skip_blanks: true}.tap do | opts|
+          opts[:col_sep] = "\t" if @store.format == :tsv
+        end
+      end
+
+      def safe_row(data)
+        row = Masamune::Schema::Row.new(parent: @table, values: data.to_hash, strict: false)
+        row.to_hash
+      rescue
+        @store.logger.warn("failed to parse '#{data.to_hash}' for #{@table.name}, skipping")
+      end
+
+      def append?(elem)
+        return true unless @options[:distinct]
+        @seen ||= Set.new
+        @seen.add?(elem)
+      end
+    end
+
+    DEFAULT_ATTRIBUTES =
+    {
+      source:    nil,
+      target:    nil,
+      columns:   nil,
+      store:     nil,
+      function:  ->(row) { row },
+      distinct:  false,
+      debug:     false
+    }
+
+    DEFAULT_ATTRIBUTES.keys.each do |attr|
+      attr_accessor attr
+    end
+
+    def initialize(opts = {})
+      opts.symbolize_keys!
+      raise ArgumentError, 'required parameter source: missing' unless opts.key?(:source)
+      raise ArgumentError, 'required parameter target: missing' unless opts.key?(:target)
+      DEFAULT_ATTRIBUTES.merge(opts).each do |name, value|
+        public_send("#{name}=", value)
+      end
+    end
+
+    def source=(source)
+      @source = source
+    end
+
+    # FIXME: avoid implict conversions
+    def target=(target)
+      @target = target.type == :four ? target.ledger_table : target
+    end
+
+    def intermediate_columns
+      output = function.call(default_row(source.columns))
+      example = Array.wrap(output).first
+      raise ArgumentError, "function for map between '#{source.name}' and '#{target.name}' does not return output for default input" unless example
+      example.keys
+    end
+
+    def intermediate
+      target.stage_table(columns: columns || intermediate_columns, inherit: false)
+    end
+
+    def apply(input_files, output_file)
+      input_buffer  = Buffer.new(source)
+      output_buffer = Buffer.new(intermediate, distinct: distinct)
+      self.class.convert_files(input_files).each do |input_file|
+        open_stream(input_file, 'r') do |input_stream|
+          input_buffer.bind(input_stream)
+          open_stream(output_file, 'a+') do |output_stream|
+            output_buffer.bind(output_stream)
+            apply_buffer(input_buffer, output_buffer)
+          end
+        end
+      end
+      intermediate
+    end
+
+    def open_stream(file, mode, &block)
+      case file
+      when IO, StringIO
+        file.flush
+        yield file
+      when String, Tempfile
+        File.open(file, mode) do |io|
+          yield io
+        end
+      end
+    end
+
+    class << self
+      def convert_file(file)
+        if file.respond_to?(:path)
+          file.flush if file.respond_to?(:flush) && file.respond_to?(:open?) && file.open?
+          file.path
+        else
+          file
+        end
+      end
+
+      def convert_files(files)
+        case files
+        when Set
+          files.map { |file| convert_file(file) }.to_a
+        when Array
+          files.map { |file| convert_file(file) }.to_a
+        else
+          [convert_file(files)]
+        end
+      end
+    end
+
+    private
+
+    def default_row(columns)
+      {}.tap do |row|
+        columns.each do |_, column|
+          row[column.name] = column.default_ruby_value
+        end
+      end
+    end
+
+    def apply_buffer(input_buffer, output_buffer)
+      input_buffer.each do |input|
+        safe_apply_function(input) do |output|
+          output_buffer.append output
+        end
+      end
+      output_buffer.flush
+    end
+
+    def safe_apply_function(input, &block)
+      return unless input
+      Array.wrap(function.call(input)).each do |output|
+        yield output
+      end
+    rescue
+      @store.logger.warn("failed to process '#{input}' for #{target.name}, skipping")
+    end
+  end
+end