masamune 0.11.0

data/lib/masamune/has_environment.rb

@@ -0,0 +1,40 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'delegate'
+require 'forwardable'
+
+module Masamune
+  module HasEnvironment
+    extend Forwardable
+
+    def environment
+      @environment ||= Masamune::Environment.new
+    end
+
+    def environment=(environment)
+      @environment = environment
+    end
+
+    def_delegators :environment, :configure, :configuration, :with_exclusive_lock, :logger, :log_file_name, :filesystem, :filesystem=, :catalog, :trace, :console, :hive_helper, :postgres_helper
+  end
+end

data/lib/masamune/helpers.rb

@@ -0,0 +1,27 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune
+  module Helpers
+    require 'masamune/helpers/postgres'
+  end
+end

data/lib/masamune/helpers/postgres.rb

@@ -0,0 +1,84 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'masamune/has_environment'
+require 'masamune/actions/postgres'
+
+module Masamune::Helpers
+  class Postgres
+    include Masamune::HasEnvironment
+    include Masamune::Actions::Postgres
+    include Masamune::Actions::PostgresAdmin
+
+    def initialize(environment)
+      self.environment = environment
+      clear!
+    end
+
+    def clear!
+      @cache = {}
+    end
+
+    def database_exists?
+      postgres(exec: 'SELECT version();', fail_fast: false).success?
+    end
+
+    def table_exists?(table)
+      return unless database_exists?
+      tables.include?(table)
+    end
+
+    def table_last_modified_at(table, options = {})
+      column = options[:last_modified_at]
+      return unless column
+      return unless table_exists?(table)
+      update_table_last_modified_at(table, column)
+      @cache[table]
+    end
+
+    def tables
+      update_tables
+      @cache.keys
+    end
+
+    private
+
+    def update_tables
+      return unless @cache.empty?
+      postgres(exec: 'SELECT table_name FROM information_schema.tables;', tuple_output: true) do |line|
+        table = line.strip
+        next if table =~ /\Apg_/
+        @cache[table] ||= nil
+      end
+    end
+
+    def update_table_last_modified_at(table, column)
+      return if @cache[table].present?
+      postgres(exec: "SELECT MAX(#{column}) FROM #{table};", tuple_output: true) do |line|
+        begin
+          @cache[table] = Time.parse(line.strip).at_beginning_of_minute.utc
+        rescue ArgumentError
+        end
+      end
+    end
+  end
+end

data/lib/masamune/io.rb

@@ -0,0 +1,33 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+class IO
+  def wait_readable(timeout = 0)
+    rs, ws = IO.select([self], [], [], timeout)
+    rs && rs[0]
+  end unless IO.method_defined?(:wait_readable)
+
+  def wait_writable(timeout = 0)
+    rs, ws = IO.select([], [self], [], timeout)
+    ws && ws[0]
+  end unless IO.method_defined?(:wait_writable)
+end

data/lib/masamune/last_element.rb

@@ -0,0 +1,53 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'active_support/concern'
+
+module Masamune
+  module LastElement
+    extend ActiveSupport::Concern
+
+    def last_element(method, *args)
+      instance = send(method, *args)
+      case instance
+      when Array
+        instance.map { |elem| [elem, elem == instance.last] }
+      when Hash
+        instance.map { |key, value| [key, value, key == instance.keys.last] }
+      end
+    end
+
+    module ClassMethods
+      def method_with_last_element(method)
+        self.class_eval do
+          new_method = "#{method}_with_last_element"
+          old_method = "#{method}_without_last_element"
+          alias_method old_method, method
+          define_method(new_method) do |*args, &block|
+            last_element(old_method,  *args)
+          end
+          alias_method method, new_method
+        end
+      end
+    end
+  end
+end

data/lib/masamune/method_logger.rb

@@ -0,0 +1,41 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+# FIXME refactor as Command
+module Masamune
+  class MethodLogger < SimpleDelegator
+    def initialize(target, *methods)
+      super target
+      @target = target
+      @methods = methods
+    end
+
+    def method_missing(method_name, *args, &block)
+      @target.environment.console("#{method_name} with #{args.join(' ')}") if @methods.include?(method_name)
+      if @target.respond_to?(method_name)
+        @target.__send__(method_name, *args, &block)
+      else
+        super
+      end
+    end
+  end
+end

data/lib/masamune/multi_io.rb

@@ -0,0 +1,39 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+# From: http://stackoverflow.com/a/6407200
+module Masamune
+  class MultiIO
+    def initialize(*targets)
+      @targets = targets
+      @targets.each { |t| t.sync = true }
+    end
+
+    def write(*args)
+      @targets.each { |t| t.write(*args) }
+    end
+
+    def close
+      @targets.each(&:close)
+    end
+  end
+end

data/lib/masamune/schema.rb

@@ -0,0 +1,36 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune
+  module Schema
+    require 'masamune/schema/catalog'
+    require 'masamune/schema/store'
+    require 'masamune/schema/table'
+    require 'masamune/schema/table_reference'
+    require 'masamune/schema/dimension'
+    require 'masamune/schema/fact'
+    require 'masamune/schema/column'
+    require 'masamune/schema/row'
+    require 'masamune/schema/event'
+    require 'masamune/schema/map'
+  end
+end

data/lib/masamune/schema/catalog.rb

@@ -0,0 +1,233 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'active_support/core_ext/hash'
+
+require 'masamune/schema/store'
+
+module Masamune::Schema
+  class Catalog
+    include Masamune::HasEnvironment
+
+    class HasMap < SimpleDelegator
+      attr_accessor :maps
+
+      def initialize(*args)
+        super
+        @maps = {}
+      end
+
+      def map(options = {})
+        self.maps[options[:to]]
+      end
+    end
+
+    class HasFormat < SimpleDelegator
+      def initialize(store, options = {})
+        super store
+        @options = options
+      end
+
+      def format
+        @options.key?(:format) ? @options[:format] : __getobj__.format
+      end
+
+      def json_encoding
+        @options.key?(:json_encoding) ? @options[:json_encoding] : __getobj__.json_encoding
+      end
+
+      def headers
+        @options.key?(:headers) ? @options[:headers] : __getobj__.headers
+      end
+    end
+
+    class Context < SimpleDelegator
+      attr_accessor :options
+
+      def initialize(store, options = {})
+        super store
+        @store    = store
+        @options  = Hash.new { |h,k| h[k] = [] }
+        @options.merge!(store: @store)
+        @options.merge!(options)
+      end
+
+      def push(options = {})
+        @prev_options = @options.dup
+        @options.merge!(options)
+      end
+
+      def pop
+        @options = @prev_options
+      end
+    end
+
+    def initialize(environment)
+      self.environment = environment
+      @stores   = Hash.new { |h,k| h[k] = Masamune::Schema::Store.new(environment, type: k) }
+      @context  = nil
+    end
+
+    def clear!
+      @stores.clear
+    end
+
+    def schema(*args, &block)
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      raise ArgumentError, 'schema store arguments required' unless args.any?
+      stores = args.map(&:to_sym)
+      stores.each do |id|
+        begin
+          @context = Context.new(@stores[id], options)
+          instance_eval &block
+        ensure
+          @context = nil
+        end
+      end
+    end
+
+    Masamune::Schema::Store.types.each do |store|
+      define_method(store) do
+        @stores[store]
+      end
+    end
+
+    def [](store_id)
+      @stores[store_id.to_sym]
+    end
+
+    def table(id, options = {}, &block)
+      @context.push(options)
+      yield if block_given?
+      @context.tables[id] ||= Masamune::Schema::Table.new(@context.options.merge(id: id))
+      @context.references[id] ||= Masamune::Schema::TableReference.new(@context.tables[id])
+    ensure
+      @context.pop
+    end
+
+    def dimension(id, options = {}, &block)
+      @context.push(options)
+      yield if block_given?
+      @context.dimensions[id] ||= Masamune::Schema::Dimension.new(@context.options.merge(id: id))
+      @context.references[id] ||= Masamune::Schema::TableReference.new(@context.dimensions[id])
+    ensure
+      @context.pop
+    end
+
+    def column(id, options = {}, &block)
+      @context.options[:columns] << dereference_column(id, options)
+    end
+
+    # FIXME: references should not be ambiguous, e.g. references :user, should be references :user_dimension
+    def references(id, options = {})
+      table = @context.tables[id] || @context.dimensions[id]
+      table ||= Masamune::Schema::Dimension.new(id: id, type: :mini) if options[:degenerate]
+      reference = Masamune::Schema::TableReference.new(table, options.reverse_merge(denormalize: table.implicit))
+      @context.references[reference.id] = reference
+      @context.options[:references] << reference
+    end
+
+    def row(options)
+      attributes = options.delete(:attributes) || {}
+      attributes[:values] = options
+      @context.options[:rows] << Masamune::Schema::Row.new(attributes)
+    end
+
+    def fact(id, options = {}, &block)
+      @context.push(options)
+      grain = Array.wrap(options.delete(:grain) || [])
+      fact_attributes(grain).each do |attributes|
+        yield if block_given?
+        table = Masamune::Schema::Fact.new(@context.options.merge(id: id).merge(attributes))
+        @context.facts[table.id] ||= table
+      end
+    ensure
+      @context.pop
+    end
+
+    def partition(id, options = {}, &block)
+      @context.options[:columns] << Masamune::Schema::Column.new(options.merge(id: id, partition: true))
+    end
+
+    def measure(id, options = {}, &block)
+      @context.options[:columns] << Masamune::Schema::Column.new(options.merge(id: id, measure: true))
+    end
+
+    def file(id, options = {}, &block)
+      format_options = options.extract!(:format, :headers)
+      @context.push(options)
+      yield if block_given?
+      store = HasFormat.new(@context, format_options)
+      @context.files[id] = HasMap.new Masamune::Schema::Table.new(@context.options.merge(id: id, type: :stage, store: store))
+    ensure
+      @context.pop
+    end
+
+    def event(id, options = {}, &block)
+      @context.push(options)
+      yield if block_given?
+      @context.events[id] = HasMap.new Masamune::Schema::Event.new(@context.options.merge(id: id))
+    ensure
+      @context.pop
+    end
+
+    def attribute(id, options = {}, &block)
+      @context.options[:attributes] << Masamune::Schema::Event::Attribute.new(options.merge(id: id))
+    end
+
+    def map(options = {}, &block)
+      raise ArgumentError, "invalid map, from: is missing" unless options.is_a?(Hash)
+      from, to = options.delete(:from), options.delete(:to)
+      raise ArgumentError, "invalid map, from: is missing" unless from && from.try(:id)
+      raise ArgumentError, "invalid map from: '#{from.id}', to: is missing" unless to
+      @context.push(options)
+      @context.options[:function] = block.to_proc
+      from.maps[to] ||= Masamune::Schema::Map.new(@context.options.merge(source: from, target: to))
+    ensure
+      @context.pop
+    end
+
+    def load(file)
+      case file
+      when /\.rb\Z/
+        instance_eval(File.read(file), file)
+      when /\.psql\Z/
+        @stores[:postgres].extra << file
+      when /\.hql\Z/
+        @stores[:hive].extra << file
+      end
+    end
+
+    private
+
+    def dereference_column(id, options = {})
+      store_id = id.split(/\./).reverse.last.try(:to_sym)
+      context = store_id && @stores.key?(store_id) ? @stores[store_id] : @context
+      context.dereference_column(id, options)
+    end
+
+    def fact_attributes(grain = [])
+      return [{}] unless grain.any?
+      grain.map { |x| { grain: x } }
+    end
+  end
+end