masamune 0.11.0

data/lib/masamune/schema/table_reference.rb

@@ -0,0 +1,76 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Schema
+  class TableReference < SimpleDelegator
+    DEFAULT_ATTRIBUTES =
+    {
+      label:           nil,
+      insert:          false,
+      null:            false,
+      default:         nil,
+      natural_key:     false,
+      denormalize:     false,
+      multiple:        false,
+      degenerate:      false
+    }
+
+    DEFAULT_ATTRIBUTES.keys.each do |attr|
+      attr_accessor attr
+    end
+
+    def initialize(table, opts = {})
+      super table
+      @table = table
+      DEFAULT_ATTRIBUTES.merge(opts).each do |name, value|
+        public_send("#{name}=", value)
+      end
+    end
+
+    def id
+      [label, @table.id].compact.join('_').to_sym
+    end
+
+    def alias
+      [label, @table.name].compact.join('_')
+    end
+
+    def foreign_key
+      @table.surrogate_key
+    end
+
+    def foreign_key_name
+      [label, @table.name, foreign_key.name].compact.join('_').to_sym
+    end
+
+    def foreign_key_type
+      foreign_key.type
+    end
+
+    def default(column = nil)
+      return if @default == :null
+      if default_row = @table.rows.detect { |row| @default ? row.id == @default : row.default }
+        default_row.name(column)
+      end
+    end
+  end
+end

data/lib/masamune/spec_helper.rb

@@ -0,0 +1,23 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+Dir["#{File.dirname(__FILE__)}/../../spec/support/masamune/**/*.rb"].each {|f| require f}

data/lib/masamune/string_format.rb

@@ -0,0 +1,34 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune
+  module StringFormat
+    def strip_sql(sql)
+      out = sql.dup
+      out.gsub!(/\A'|\A"|"\z|'\z/, '')
+      out.gsub!(/\s\s+/, ' ')
+      out.gsub!(/\s*;+\s*$/,'')
+      out.strip!
+      out + ';'
+    end
+  end
+end

data/lib/masamune/tasks/elastic_mapreduce_thor.rb

@@ -0,0 +1,60 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'masamune'
+require 'thor'
+
+module Masamune::Tasks
+  class ElasticMapreduceThor < Thor
+    include Masamune::Thor
+    include Masamune::Actions::ElasticMapreduce
+
+    # FIXME need to add an unnecessary namespace until this issue is fixed:
+    # https://github.com/wycats/thor/pull/247
+    namespace :elastic_mapreduce
+
+    desc 'elastic_mapreduce', 'Launch an ElasticMapReduce ssh session'
+    class_option :template, :type => :string, :aliases => '-t', :desc => 'Execute named template command'
+    class_option :params, :type => :hash, :aliases => '-p', :desc => 'Bind params to named template command', :default => {}
+    def elastic_mapreduce_exec
+      elastic_mapreduce_options = options.dup.with_indifferent_access
+      elastic_mapreduce_options.merge!(interactive: true)
+      elastic_mapreduce_options.merge!(extra: extra_or_ssh)
+      elastic_mapreduce(elastic_mapreduce_options)
+    end
+    default_task :elastic_mapreduce_exec
+
+    no_tasks do
+      after_initialize(:first) do |thor, options|
+        begin
+          thor.extra += thor.configuration.bind_template(:elastic_mapreduce, options[:template], options[:params]) if options[:template]
+        rescue ArgumentError => e
+          raise ::Thor::MalformattedArgumentError, e.to_s
+        end
+      end
+
+      def extra_or_ssh
+        self.extra.any? ? self.extra : ['--ssh']
+      end
+    end
+  end
+end

data/lib/masamune/tasks/hive_thor.rb

@@ -0,0 +1,55 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'masamune'
+require 'thor'
+
+module Masamune::Tasks
+  class HiveThor < Thor
+    include Masamune::Thor
+    include Masamune::Actions::ElasticMapreduce
+    include Masamune::Actions::Hive
+
+    # FIXME need to add an unnecessary namespace until this issue is fixed:
+    # https://github.com/wycats/thor/pull/247
+    namespace :hive
+
+    desc 'hive', 'Launch a Hive session'
+    method_option :file, :aliases => '-f', :desc => 'SQL from files'
+    method_option :exec, :aliases => '-e', :desc => 'SQL from command line'
+    method_option :output, :aliases => '-o', :desc => 'Save SQL output to file'
+    method_option :delimiter, :desc => 'Hive row format delimiter', :default => "\001"
+    method_option :csv, :type => :boolean, :desc => 'Report SQL output in CSV format', :default => false
+    method_option :variables, :aliases => '-X', :type => :hash, :desc => 'Variables to substitute in SQL', :default => {}
+    method_option :retry, :type => :boolean, :desc => 'Retry SQL query in event of failure', :default => false
+    method_option :service, :desc => 'Start as a service', :default => false
+    def hive_exec
+      hive_options = options.dup.with_indifferent_access
+      hive_options.merge!(print: true)
+      hive_options.merge!(retries: 0) unless options[:retry]
+      hive_options.merge!(file: File.expand_path(options[:file])) if options[:file]
+      hive_options.merge!(output: File.expand_path(options[:output])) if options[:output]
+      hive(hive_options)
+    end
+    default_task :hive_exec
+  end
+end

data/lib/masamune/tasks/postgres_thor.rb

@@ -0,0 +1,47 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'masamune'
+require 'thor'
+
+module Masamune::Tasks
+  class PostgresThor < Thor
+    include Masamune::Thor
+    include Masamune::Actions::Postgres
+
+    # FIXME need to add an unnecessary namespace until this issue is fixed:
+    # https://github.com/wycats/thor/pull/247
+    namespace :postgres
+
+    desc 'psql', 'Launch a Postgres session'
+    method_option :file, :aliases => '-f', :desc => 'SQL from files'
+    method_option :exec, :aliases => '-e', :desc => 'SQL from command line'
+    method_option :output, :aliases => '-o', :desc => 'Save SQL output to file'
+    method_option :csv, :type => :boolean, :desc => 'Report SQL output in CSV format', :default => false
+    def psql_exec
+      postgres_options = options.dup.with_indifferent_access
+      postgres_options.merge!(print: true)
+      postgres(postgres_options)
+    end
+    default_task :psql_exec
+  end
+end

data/lib/masamune/tasks/shell_thor.rb

@@ -0,0 +1,63 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'masamune'
+require 'thor'
+require 'pry'
+
+module Masamune::Tasks
+  class ShellThor < Thor
+    include Masamune::Thor
+    include Masamune::Actions::DataFlow
+    include Masamune::Transform::DefineSchema
+
+    # FIXME need to add an unnecessary namespace until this issue is fixed:
+    # https://github.com/wycats/thor/pull/247
+    namespace :shell
+
+    desc 'shell', 'Launch an interactive shell'
+    method_option :dump, :type => :boolean, :desc => 'Dump SQL schema', :default => false
+    method_option :type, :enum => ['psql', 'hql'], :desc => 'Schema type', :default => 'psql'
+    method_option :prompt, :desc => 'Set shell prompt', :default => 'masamune'
+    class_option :start, :aliases => '-a', :desc => 'Start time', default: '1 month ago'
+    def shell_exec
+      if options[:dump]
+        print_catalog
+        exit
+      end
+
+      Pry.start self, prompt: proc { options[:prompt] + '> ' }
+    end
+    default_task :shell_exec
+
+    private
+
+    def print_catalog
+      case options[:type]
+      when 'psql'
+        puts define_schema(catalog, :postgres)
+      when 'hql'
+        puts define_schema(catalog, :hive)
+      end
+    end
+  end
+end

data/lib/masamune/template.rb

@@ -0,0 +1,77 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'tilt'
+require 'pathname'
+
+module Masamune
+  class Template
+    def initialize(paths = [])
+      @paths = Array.wrap(paths)
+    end
+
+    def render(template, parameters = {})
+      resolved_template = resolve_file(template)
+      self.class.combine Tilt.new(resolved_template, nil, trim: '->').render(self, parameters)
+    end
+
+    private
+
+    # TODO unify with resolve_path
+    def resolve_file(partial_file)
+      return partial_file if Pathname.new(partial_file).absolute?
+      @paths.each do |path|
+        file = File.expand_path(File.join(path, partial_file))
+        return file if File.exists?(file) && File.file?(file)
+      end
+      raise IOError, "File not found: #{partial_file}"
+    end
+
+    class << self
+      def render_to_file(template, parameters = {})
+        raise IOError, "File not found: #{template}" unless File.exists?(template)
+        Tempfile.new('masamune').tap do |file|
+          file.write(render_to_string(template, parameters))
+          file.close
+        end.path
+      end
+
+      def render_to_string(template, parameters = {})
+        raise IOError, "File not found: #{template}" unless File.exists?(template)
+        instance = Template.new(File.dirname(template))
+        combine instance.render(template, parameters)
+      end
+
+      def combine(*a)
+        strip_newlines(strip_comments(a.join("\n"), /^--.*$/)) + "\n"
+      end
+
+      def strip_newlines(s)
+        s.gsub(/^\n+/, "\n").lstrip.strip
+      end
+
+      def strip_comments(s, comment)
+        strip_newlines s.gsub(comment, '')
+      end
+    end
+  end
+end