masamune 0.14.0 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: feca9f373a42ecd946e9682b08213c3357ce7437
-  data.tar.gz: 51cf1a4d353fb1f607299937e60c66437213d35c
+  metadata.gz: 7abad0d40705fd8c85b74398cf3ce198d6e1e0a3
+  data.tar.gz: 049aeb65a25a01b8dca0a9db21d7e543e9eadae5
 SHA512:
-  metadata.gz: 2e1d3c79fbe068bb79cb653cb746845dbaf7c33bc4525727cfa569b40407578b970418ee701ecb4ed0ed15004fb48d183c14398c03e6267fa8f5429d0184d0cf
-  data.tar.gz: f62cd40539a9ac2036ee312e9abf9055792463f3340faaed9857bd2dfb16d06cf3fe12e32ba41bc1b3676b23e512641abcaf07c929fe3e0e70bc5b4240d002ea
+  metadata.gz: 5a8ef99944a527c5d0fde7b4875fc1d6541dd830060372fb47752a8a915f8d6bb7a3c359ad54c2592b2939c1bbc0ac1ea2c486f34c75fb31c7551524ef278963
+  data.tar.gz: 3d44f7583a6048a70010c124b8f8c881163196b285ced0fbfe78cb0bc3858bcd3768bc3b3528f0bd8a8a9043060c8fb33f4a813bb472ffa3a751cd71bd0fbbbb

data/README.md

@@ -26,7 +26,7 @@ class ExampleThor  < Thor
   # Describe a Data Processing Job
   desc 'extract_logs', 'Organize log files by YYYY-MM-DD'
 
-  target fs.path(:target_dir, '%Y-%m-%d', mkdir: true)
+  target fs.path(:target_dir, '%Y-%m-%d')
   source fs.path(:source_dir, '%Y%m%d*.log')
   def extract_logs
     targets.missing.each do |target|
@@ -45,6 +45,15 @@ Execute your dataflow with the goal of processing all data from the start of the
 thor extract_logs --start '1 year ago'
 ```
 
+Testing
+---------------
+```
+rake spec             # Run Rspec unit code examples
+rake spec:acceptance  # Run Rspec acceptance code examples
+rake spec:all         # Run All Rspec code examples
+rake spec:unit        # Run Rspec unit code examples
+```
+
 Contributing
 ---------------
 

data/Rakefile

@@ -1,4 +1,25 @@
 #!/usr/bin/env rake
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
 
 begin
   require 'bundler/setup'
@@ -10,6 +31,22 @@ Bundler::GemHelper.install_tasks
 
 require 'rspec/core/rake_task'
 
+desc 'Run Rspec unit code examples'
 RSpec::Core::RakeTask.new(:spec)
 
+namespace :spec do
+  desc 'Run Rspec unit code examples'
+  RSpec::Core::RakeTask.new(:unit) do |spec|
+    spec.pattern = "spec/**/*_spec.rb"
+  end
+
+  desc 'Run Rspec acceptance code examples'
+  RSpec::Core::RakeTask.new(:acceptance) do |spec|
+    spec.pattern = "examples/**/*_spec.rb"
+  end
+
+  desc 'Run All Rspec code examples'
+  task all: [:unit, :acceptance]
+end
+
 task :default => :spec

data/bin/masamune-dump

@@ -1,4 +1,26 @@
 #!/usr/bin/env ruby
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
 $: << File.expand_path('../../lib/', __FILE__)
 require 'masamune/tasks/dump_thor'
 Masamune::Tasks::DumpThor.start(ARGV)

data/bin/masamune-elastic-mapreduce

@@ -1,4 +1,26 @@
 #!/usr/bin/env ruby
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
 $: << File.expand_path('../../lib/', __FILE__)
 require 'masamune/tasks/elastic_mapreduce_thor'
 Masamune::Tasks::ElasticMapreduceThor.start(ARGV)

data/bin/masamune-hive

@@ -1,4 +1,26 @@
 #!/usr/bin/env ruby
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
 $: << File.expand_path('../../lib/', __FILE__)
 require 'masamune/tasks/hive_thor'
 Masamune::Tasks::HiveThor.start(ARGV)

data/bin/masamune-psql

@@ -1,4 +1,26 @@
 #!/usr/bin/env ruby
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
 $: << File.expand_path('../../lib/', __FILE__)
 require 'masamune/tasks/postgres_thor'
 Masamune::Tasks::PostgresThor.start(ARGV)

data/bin/masamune-shell

@@ -1,4 +1,26 @@
 #!/usr/bin/env ruby
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
 $: << File.expand_path('../../lib/', __FILE__)
 require 'masamune/tasks/shell_thor'
 Masamune::Tasks::ShellThor.start(ARGV)

data/lib/masamune/cached_filesystem.rb

@@ -33,7 +33,7 @@ module Masamune
     end
 
     def exists?(file)
-      @cache.key?(file) || glob(file).include?(file) || @cache.key?(file)
+      @cache[file].any? || glob(file).include?(file) || @cache[file].any?
     end
 
     def glob(file_or_glob)

data/lib/masamune/commands/shell.rb

@@ -75,7 +75,7 @@ module Masamune::Commands
     end
 
     def around_execute(&block)
-      if configuration.no_op && !safe
+      if configuration.dry_run && !safe
         return OpenStruct.new(:success? => true)
       end
 

data/lib/masamune/configuration.rb

@@ -36,7 +36,6 @@ class Masamune::Configuration
   attr_accessor :quiet
   attr_accessor :verbose
   attr_accessor :debug
-  attr_accessor :no_op
   attr_accessor :dry_run
   attr_accessor :lock
   attr_accessor :retries
@@ -56,7 +55,6 @@ class Masamune::Configuration
     self.quiet    = false
     self.verbose  = false
     self.debug    = false
-    self.no_op    = false
     self.dry_run  = false
     self.lock     = nil
     self.retries  = 3
@@ -70,9 +68,10 @@ class Masamune::Configuration
     end
   end
 
-  def load(file)
+  def load(path)
     @load_once ||= begin
-      load_yaml_erb_file(file).each_pair do |command, value|
+      config_file = filesystem.eval_path(path)
+      load_yaml_erb_file(config_file).each_pair do |command, value|
         if COMMANDS.include?(command)
           send("#{command}=", value)
         elsif command == 'paths'
@@ -82,7 +81,7 @@ class Masamune::Configuration
           self.params.merge! value
         end
       end
-      logger.debug("Loaded configuration #{file}")
+      logger.debug("Loaded configuration #{config_file}")
       load_catalog(configuration.postgres.fetch(:schema_files, []) + configuration.hive.fetch(:schema_files, []))
       true
     end
@@ -132,8 +131,7 @@ class Masamune::Configuration
     opts << '--quiet'   if quiet
     opts << '--verbose' if verbose
     opts << '--debug'   if debug
-    opts << '--no_op'   if no_op
-    opts << '--dry_run' if dry_run
+    opts << '--dry-run' if dry_run
     opts
   end
 

data/lib/masamune/data_plan/elem.rb

@@ -47,10 +47,8 @@ class Masamune::DataPlan::Elem
   def exists?
     if rule.for_path?
       rule.engine.filesystem.exists?(path)
-    elsif rule.for_table_with_partition?
-      rule.engine.postgres_helper.table_exists?(table)
     elsif rule.for_table?
-      table
+      rule.engine.postgres_helper.table_exists?(table)
     end
   end
 
@@ -70,14 +68,16 @@ class Masamune::DataPlan::Elem
 
   def explode
     return Set.new(to_enum(__method__)) unless block_given?
-    if rule.for_path?
+    if rule.for_path? && rule.free?
       file_glob = path
       file_glob += '/' unless path.include?('*') || path.include?('.')
       file_glob += '*' unless path.include?('*')
-      rule.engine.filesystem.glob(file_glob).each do |new_path|
+      rule.engine.filesystem.glob(file_glob) do |new_path|
         yield rule.bind_input(new_path)
       end
-    elsif rule.for_table_with_partition?
+    elsif rule.for_path? && rule.bound?
+      yield self if exists?
+    elsif rule.for_table?
       yield self if exists?
     end
   end
@@ -136,6 +136,10 @@ class Masamune::DataPlan::Elem
     @options[:glob]
   end
 
+  def rest
+    @options[:rest]
+  end
+
   def next(i = 1)
     self.class.new(@rule, start_time.advance(@rule.time_step => +1*i), @options)
   end
@@ -184,6 +188,10 @@ class Masamune::DataPlan::Elem
   private
 
   def strftime_format
-    @strftime_format ||= glob ? @rule.strftime_format.sub('*', glob) : @rule.strftime_format
+    @strftime_format ||= begin
+      format = @rule.strftime_format.dup
+      format.sub!('*', glob || rest) if glob || rest
+      format
+    end
   end
 end

data/lib/masamune/data_plan/engine.rb

@@ -121,6 +121,8 @@ class Masamune::DataPlan::Engine
   def prepare(rule, options = {})
     @targets[rule].merge options.fetch(:targets, [])
     @sources[rule].merge options.fetch(:sources, [])
+    @target_rules[rule].try(:prepare)
+    @source_rules[rule].try(:prepare)
 
     constrain_max_depth(rule) do
       sources(rule).group_by { |source| rule_for_target(source.input) }.each do |derived_rule, sources|

data/lib/masamune/data_plan/rule.rb

@@ -40,6 +40,10 @@ class Masamune::DataPlan::Rule
     @options = options
   end
 
+  def prepare
+    pattern
+  end
+
   def for_targets?
     @type == :target
   end
@@ -91,7 +95,7 @@ class Masamune::DataPlan::Rule
   def pattern
     @pattern ||= begin
       if for_path?
-        path.respond_to?(:call) ? path.call(engine.filesystem) : path
+        engine.filesystem.eval_path(path)
       elsif for_table_with_partition?
         [table , partition].join('_')
       elsif for_table?
@@ -104,9 +108,17 @@ class Masamune::DataPlan::Rule
     @options.fetch(:primary, true)
   end
 
+  def free?
+    pattern.include?('%') || pattern.include?('*')
+  end
+
+  def bound?
+    !free?
+  end
+
   def matches?(input)
     matched_pattern = match_data_hash(matcher.match(input))
-    matched_pattern.present? && matched_pattern[:rest].blank?
+    matched_pattern.present? && (matched_pattern[:rest].blank? || matched_pattern[:rest].include?('*'))
   end
 
   def bind_date_or_time(input = nil)
@@ -288,9 +300,8 @@ class Masamune::DataPlan::Rule
     end
   end
 
-  def matched_extra(matched_data)
-    return {} unless matched_data.has_key?(:glob)
-    {glob: matched_data[:glob]}.reject { |_,v| v == '*' }
+  def matched_extra(matched_data = {})
+    matched_data.slice(:glob, :rest).reject { |_, v| v.blank? || v == '*' }
   end
 
   def options_for_elem

data/lib/masamune/data_plan/set.rb

@@ -47,7 +47,7 @@ class Masamune::DataPlan::Set < Set
   def missing
     return self.class.new(rule, to_enum(__method__)) unless block_given?
     self.each do |elem|
-      yield elem if elem.explode.count < 1
+      yield elem if elem.explode.none?
     end
   end
 
@@ -90,13 +90,13 @@ class Masamune::DataPlan::Set < Set
     return Masamune::DataPlan::Set::EMPTY if empty? || @rule.for_sources?
     return self.class.new(rule, to_enum(__method__)) unless block_given?
     set = Set.new
-    missing.each do |target|
+    missing do |target|
       yield target if set.add?(target)
     end
-    incomplete.each do |target|
+    incomplete do |target|
       yield target if set.add?(target)
     end
-    stale.each do |target|
+    stale do |target|
       yield target if set.add?(target)
     end
   end
@@ -105,7 +105,7 @@ class Masamune::DataPlan::Set < Set
     return Masamune::DataPlan::Set::EMPTY if empty? || @rule.for_sources?
     return self.class.new(rule, to_enum(__method__)) unless block_given?
     set = Set.new
-    actionable.each do |target|
+    actionable do |target|
       yield target if set.add?(target) && target.sources.existing.any?
     end
   end
@@ -124,7 +124,7 @@ class Masamune::DataPlan::Set < Set
     return Masamune::DataPlan::Set::EMPTY if empty? || @rule.for_targets?
     return self.class.new(self.first.targets.rule, to_enum(__method__)) unless block_given?
     self.each do |elem|
-      elem.targets.each do |target|
+      elem.targets do |target|
         yield target
       end
     end
@@ -134,7 +134,7 @@ class Masamune::DataPlan::Set < Set
     return Masamune::DataPlan::Set::EMPTY if empty? || @rule.for_sources?
     return self.class.new(self.first.sources.rule, to_enum(__method__)) unless block_given?
     self.each do |elem|
-      elem.sources.each do |source|
+      elem.sources do |source|
         yield source
       end
     end
@@ -171,6 +171,6 @@ class Masamune::DataPlan::Set < Set
   def target_stale?(source, target)
     target.last_modified_at != Masamune::DataPlan::Elem::MISSING_MODIFIED_AT &&
     source.last_modified_at != Masamune::DataPlan::Elem::MISSING_MODIFIED_AT &&
-    source.last_modified_at >= target.last_modified_at
+    source.last_modified_at > target.last_modified_at
   end
 end

data/lib/masamune/filesystem.rb

@@ -51,13 +51,14 @@ module Masamune
       self
     end
 
-    def get_path(symbol, *extra)
+    def get_path(symbol, *args)
+      options = args.last.is_a?(Hash) ? args.pop : {}
       lazy_path = lambda do |fs|
         fs.has_path?(symbol) or raise "Path :#{symbol} not defined"
         path, options = fs.paths[symbol]
 
         mkdir!(path) if options[:mkdir]
-        expand_params(fs, extra.any? ? File.join(path, extra) : path)
+        expand_params(fs, args.any? ? File.join(path, args) : path)
       end
 
       if eager_load_paths?
@@ -76,6 +77,10 @@ module Masamune
       @paths
     end
 
+    def eval_path(path)
+      path.respond_to?(:call) ? path.call(self) : path
+    end
+
     def expand_params(fs, path)
       new_path = path.dup
       fs.environment.configuration.params.each do |key, value|
@@ -190,7 +195,7 @@ module Masamune
           yield OpenStruct.new(name: name, mtime: Time.parse("#{date} #{time} +0000").at_beginning_of_minute.utc, size: size.to_i)
         end
       when :local
-        Dir.glob(pattern.gsub(%r{/\*\Z}, '/**/*')) do |file|
+        Dir.glob(pattern.gsub(%r{/\*(\.\w+)?\Z}, '/**/*\1')) do |file|
           stat = File.stat(file)
           yield OpenStruct.new(name: file, mtime: stat.mtime.at_beginning_of_minute.utc, size: stat.size.to_i)
         end
@@ -216,7 +221,8 @@ module Masamune
         when :s3
           touch! *dir_set.map { |dir| File.join(dir, '.not_empty') }
         when :local
-          FileUtils.mkdir_p(dir_set, file_util_args)
+          missing_dir_set = dir_set.reject { |dir| File.exists?(dir) }
+          FileUtils.mkdir_p(missing_dir_set, file_util_args) if missing_dir_set.any?
         end
       end
     end
@@ -243,7 +249,7 @@ module Masamune
           yield q(pattern, name)
         end
       when :local
-        Dir.glob(pattern.gsub(%r{/\*\Z}, '/**/*')) do |file|
+        Dir.glob(pattern.gsub(%r{/\*(\.\w+)?\Z}, '/**/*\1')) do |file|
           yield file
         end
       end
@@ -473,7 +479,7 @@ module Masamune
     end
 
     def file_util_args
-      {noop: configuration.no_op, verbose: configuration.verbose}
+      {noop: configuration.dry_run, verbose: configuration.verbose}
     end
 
     def qualify_file(dir, file)

data/lib/masamune/schema/catalog.rb

@@ -198,21 +198,21 @@ module Masamune::Schema
       @context.pop
     end
 
-    def load(file)
-      case file
+    def load(schema_file)
+      case schema_file
       when /\.rb\Z/
-        instance_eval(File.read(file), file)
+        instance_eval(File.read(schema_file), schema_file)
       when /\.psql(\.erb)?\Z/
-        @stores[:postgres].extra << file
+        @stores[:postgres].extra << schema_file
       when /\.hql(\.erb)?\Z/
-        @stores[:hive].extra << file
+        @stores[:hive].extra << schema_file
       end
     end
 
     private
 
     def dereference_column(id, options = {})
-      store_id = id.split(/\./).reverse.last.try(:to_sym)
+      store_id = id.to_s.split(/\./).reverse.last.try(:to_sym)
       context = store_id && @stores.key?(store_id) ? @stores[store_id] : @context
       context.dereference_column(id, options)
     end

data/lib/masamune/schema/column.rb

@@ -298,7 +298,7 @@ module Masamune::Schema
 
     def default_ruby_value
       return [] if array_value?
-      return {} if hash_value?
+      return HashWithIndifferentAccess.new { |h,k| h[k] = HashWithIndifferentAccess.new(&h.default_proc) } if hash_value?
       case type
       when :date
         Date.new(0)

data/lib/masamune/schema/map.rb

@@ -76,7 +76,11 @@ module Masamune::Schema
       end
 
       def separator
-        @separator ||= (@store.format == :tsv ? "\t" : ',')
+        @separator ||=
+        case @store.format
+        when :tsv then "\t"
+        when :csv then ','
+        end
       end
     end
 
@@ -237,7 +241,7 @@ module Masamune::Schema
     private
 
     def default_row(columns)
-      {}.tap do |row|
+      {}.with_indifferent_access.tap do |row|
         columns.each do |_, column|
           row[column.compact_name] = column.default_ruby_value
         end

data/lib/masamune/schema/store.rb

@@ -33,9 +33,9 @@ module Masamune::Schema
     DEFAULT_ATTRIBUTES =
     {
       type:            nil,
-      format:          ->(store) { store.type == :postgres ? :csv : :tsv },
-      json_encoding:   ->(store) { store.type == :postgres ? :quoted : :raw },
-      headers:         ->(store) { store.type == :postgres ? true : false },
+      format:          ->(store) { default_format(store) },
+      json_encoding:   ->(store) { default_json_encoding(store) },
+      headers:         ->(store) { default_headers(store) },
       debug:           false
     }
 
@@ -111,5 +111,33 @@ module Masamune::Schema
       end
       result.to_a
     end
+
+    private
+
+    class << self
+      def default_format(store)
+        case store.type
+        when :postgres then :csv
+        when :hive then :tsv
+        else :raw
+        end
+      end
+
+      def default_headers(store)
+        return false if store.format == :raw
+        case store.type
+        when :postgres then true
+        else false
+        end
+      end
+
+      def default_json_encoding(store)
+        return :raw if store.format == :raw
+        case store.type
+        when :postgres then :quoted
+        else :raw
+        end
+      end
+    end
   end
 end

data/lib/masamune/tasks/shell_thor.rb

@@ -27,7 +27,7 @@ require 'pry'
 module Masamune::Tasks
   class ShellThor < Thor
     include Masamune::Thor
-    include Masamune::Actions::DataFlow
+    include Masamune::Actions::DateParse
 
     # FIXME need to add an unnecessary namespace until this issue is fixed:
     # https://github.com/wycats/thor/pull/247