masamune 0.15.5 → 0.15.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e5057258a406f7f889d685a6060ca42ecd5e39ab
-  data.tar.gz: c4fa4ad80317574acaeebbe8017fc4d4ed3ff20b
+  metadata.gz: 5b70cf7812702e987cc6c1f72d7f50cbb3ddde9b
+  data.tar.gz: 2d487a59b8da26e186f2eefab0f7067b5e9f950f
 SHA512:
-  metadata.gz: fcac62a5623326ed3a090de28f7355b9e6bc77a36b9959483bf53214c579ab2bffcb3cdfda3c1710b44b7fbfcb026720e7bdbc34877201bfaa3c1f06f20f5b44
-  data.tar.gz: c69d42b73d6aeb0bb7c6d0cdfcdae0330e31fc8e29192dc36143c3606f085d162a7c37fbcc2b6d631fc3d2d6dab7cf565ee9a1655afc2692c9c707fd35312291
+  metadata.gz: 91777fd4da7e0928d678ce4574f91305da2673e8bbbbeed65ce892ac612808552afa68689906f70a54c8f61d4f592b821ecb4ae9e67588bb988a6dc38e2a07b7
+  data.tar.gz: 76a2f706860eae74756e2c4b7a04e596913f3c0189e591d5c5f250681fcada14177be815f4033bede60ba1fa75a70a073d7a3bb91bac6692ab21c1ee04d8df3b

data/lib/masamune/cached_filesystem.rb

@@ -22,6 +22,9 @@
 
 module Masamune
   class CachedFilesystem < SimpleDelegator
+    MAX_DEPTH   = 10
+    EMPTY_SET   = Set.new
+
     def initialize(filesystem)
       super filesystem
       @filesystem = filesystem
@@ -29,34 +32,55 @@ module Masamune
     end
 
     def clear!
-      @cache = Hash.new { |h,k| h[k] = Set.new }
+      @cache = PathCache.new(@filesystem)
     end
 
     def exists?(file)
-      @cache[file].any? || glob(file).include?(file) || @cache[file].any?
+      glob(file, max_depth: 0).include?(file) || @cache.any?(file)
     end
 
-    def glob(file_or_glob)
-      return Set.new(to_enum(:glob, file_or_glob)) unless block_given?
-      glob_stat(file_or_glob) do |entry|
-        yield entry.name unless entry.name == dirname(file_or_glob)
+    def glob(file_or_glob, options = {})
+      return Set.new(to_enum(:glob, file_or_glob, options)) unless block_given?
+      glob_stat(file_or_glob, options) do |entry|
+        yield entry.name
       end
     end
 
     def stat(file_or_dir)
       raise ArgumentError, 'cannot contain wildcard' if file_or_dir.include?('*')
-      result = Set.new
-      glob_stat(file_or_dir) do |entry|
-        result << entry
-      end
-      result += @cache[file_or_dir]
+      result = glob_stat(file_or_dir, recursive: true)
       return unless result.any?
-      return result.first if result.size == 1
       max_time = result.map { |stat| stat.try(:mtime) }.compact.max
       sum_size = result.map { |stat| stat.try(:size) }.compact.reduce(:+)
       OpenStruct.new(name: file_or_dir, mtime: max_time, size: sum_size)
     end
 
+    def glob_stat(file_or_glob, options = {}, &block)
+      return Set.new(to_enum(:glob_stat, file_or_glob, options)) unless block_given?
+      return if file_or_glob.blank?
+      return if root_path?(file_or_glob)
+      depth = options.fetch(:depth, 0)
+      max_depth = options.fetch(:max_depth, 0)
+      return if depth > MAX_DEPTH || depth > max_depth
+
+      glob_stat(dirname(file_or_glob), depth: depth + 1, max_depth: max_depth, &block)
+
+      dirname = dirname(file_or_glob)
+      unless @cache.any?(dirname)
+        pattern = root_path?(dirname) ? file_or_glob : File.join(dirname, '*')
+        @filesystem.glob_stat(pattern) do |entry|
+          @cache.put(entry.name, entry)
+        end
+      end
+
+      file_regexp = glob_to_regexp(file_or_glob, options)
+      @cache.get(dirname).each do |entry|
+        next if entry.name == dirname
+        next unless entry.name =~ file_regexp
+        yield entry
+      end if depth == 0
+    end
+
     # FIXME cache eviction policy can be more precise
     [:touch!, :mkdir!, :copy_file_to_file, :copy_file_to_dir, :copy_dir, :remove_file, :remove_dir, :move_file_to_file, :move_file_to_dir, :move_dir, :write].each do |method|
       define_method(method) do |*args|
@@ -67,42 +91,59 @@ module Masamune
 
     private
 
-    MAX_DEPTH   = 10
-    CACHE_DEPTH = 1
-    EMPTY_SET   = Set.new
+    class PathCache
+      def initialize(filesystem)
+        @filesystem = filesystem
+        @cache = {}
+      end
 
-    def glob_stat(file_or_glob, options = {}, &block)
-      return if file_or_glob.blank?
-      return if root_path?(file_or_glob)
-      depth = options.fetch(:depth, 0)
-      return if depth > MAX_DEPTH || depth > CACHE_DEPTH
+      def put(path, entry)
+        return unless path
+        return if @filesystem.root_path?(path)
+        put(File.join(@filesystem.dirname(path), '.'), OpenStruct.new(name: @filesystem.dirname(path)))
+        paths = path_split(path)
+        elems = paths.reverse.inject(entry) { |a, n| { n => a } }
+        @cache.deep_merge!(elems)
+      end
 
-      glob_stat(dirname(file_or_glob), depth: depth + 1, &block)
+      def get(path)
+        return unless path
+        paths = path_split(path)
+        elem = paths.inject(@cache) { |level, path| level.is_a?(Hash) ? level.fetch(path, {}) : level }
+        emit(elem)
+      rescue KeyError
+        EMPTY_SET
+      end
 
-      dirname = dirname(file_or_glob)
-      unless @cache.key?(dirname)
-        pattern = root_path?(dirname) ? file_or_glob : File.join(dirname, '*')
-        @filesystem.glob_stat(pattern) do |entry|
-          recursive_paths(dirname, entry.name) do |path|
-            @cache[path] << entry
-          end
+      def any?(path)
+        if elem = get(path)
+          elem.any? { |entry| entry.name.start_with?(path) }
+        else
+          false
         end
       end
-      @cache[dirname] ||= EMPTY_SET
 
-      file_regexp = glob_to_regexp(file_or_glob)
-      @cache[dirname].each do |entry|
-        yield entry if entry.name =~ file_regexp
-      end if depth == 0
-    end
+      private
 
-    def recursive_paths(root, path, options = {}, &block)
-      depth = options.fetch(:depth, 0)
-      return if depth > MAX_DEPTH
-      return if root == path
-      yield path
-      yield dirname(path)
-      recursive_paths(root, dirname(path), depth: depth + 1, &block)
+      def emit(elem)
+        return Set.new(to_enum(:emit, elem)).flatten unless block_given?
+        case elem
+        when Array, Set
+          elem.each do |e|
+            yield emit(e)
+          end
+        when Hash
+          elem.values.each do |e|
+            yield emit(e)
+          end
+        else
+          yield elem
+        end
+      end
+
+      def path_split(path)
+        path.split('/').reject { |x| x.blank? }
+      end
     end
   end
 end

data/lib/masamune/data_plan/elem.rb

@@ -70,9 +70,8 @@ class Masamune::DataPlan::Elem
     return Set.new(to_enum(__method__)) unless block_given?
     if rule.for_path? && rule.free?
       file_glob = path
-      file_glob += '/' unless path.include?('*') || path.include?('.')
-      file_glob += '*' unless path.include?('*')
-      rule.engine.filesystem.glob(file_glob) do |new_path|
+      file_glob += '/*' unless path.include?('*') || path.include?('.')
+      rule.engine.filesystem.glob(file_glob, max_depth: rule.cache_depth) do |new_path|
         yield rule.bind_input(new_path)
       end
     elsif rule.for_path? && rule.bound?

data/lib/masamune/data_plan/rule.rb

@@ -250,6 +250,17 @@ class Masamune::DataPlan::Rule
     self.class.new(engine, name, type, options.merge(path: new_pattern))
   end
 
+  def cache_depth
+    case time_step
+    when :hour, :hours
+      2
+    when :day, :days
+      1
+    else
+      0
+    end
+  end
+
   private
 
   def time_step_to_format(step)

data/lib/masamune/filesystem.rb

@@ -111,9 +111,9 @@ module Masamune
       parent_paths.each do |part|
         tmp << part
         current_path = prefix + File.join(tmp)
-        break if current_path == path
         result << current_path
       end
+      result.pop
       result
     end
 
@@ -129,7 +129,7 @@ module Masamune
     end
 
     def dirname(path)
-      parent_paths(path).last || path
+      parent_paths(path).last || remote_prefix(path) || local_prefix(path)
     end
 
     def basename(path)
@@ -174,8 +174,8 @@ module Masamune
       end
     end
 
-    def glob_stat(pattern)
-      return Set.new(to_enum(:glob_stat, pattern)) unless block_given?
+    def glob_stat(pattern, options = {})
+      return Set.new(to_enum(:glob_stat, pattern, options)) unless block_given?
       case type(pattern)
       when :hdfs
         hadoop_fs('-ls', '-R', pattern, safe: true) do |line|
@@ -206,7 +206,6 @@ module Masamune
       raise ArgumentError, 'cannot contain wildcard' if file_or_dir.include?('*')
       result = glob_stat(file_or_dir)
       return unless result.any?
-      return result.first if result.size == 1
       max_time = result.map { |stat| stat.try(:mtime) }.compact.max
       sum_size = result.map { |stat| stat.try(:size) }.compact.reduce(:+)
       OpenStruct.new(name: file_or_dir, mtime: max_time, size: sum_size)
@@ -227,8 +226,8 @@ module Masamune
       end
     end
 
-    def glob(pattern)
-      return Set.new(to_enum(:glob, pattern)) unless block_given?
+    def glob(pattern, options = {})
+      return Set.new(to_enum(:glob, pattern, options)) unless block_given?
       case type(pattern)
       when :hdfs
         file_glob, file_regexp = glob_split(pattern)
@@ -462,7 +461,12 @@ module Masamune
       dir[%r{\Ahdfs://}]
     end
 
-    def local_prefix(file)
+    def local_prefix(dir)
+      dir[%r{\A/}] ||
+      '.'
+    end
+
+    def local_file_prefix(file)
       return file if remote_prefix(file)
       "file://#{file}"
     end
@@ -529,7 +533,7 @@ module Masamune
       when [:hdfs, :hdfs]
         hadoop_fs('-cp', src, dst)
       when [:hdfs, :local]
-        hadoop_fs('-copyToLocal', src, local_prefix(dst))
+        hadoop_fs('-copyToLocal', src, local_file_prefix(dst))
       when [:hdfs, :s3]
         hadoop_fs('-cp', src, s3n(dst))
       when [:s3, :s3]
@@ -541,7 +545,7 @@ module Masamune
       when [:local, :local]
         FileUtils.cp(src, dst, file_util_args)
       when [:local, :hdfs]
-        hadoop_fs('-copyFromLocal', local_prefix(src), dst)
+        hadoop_fs('-copyFromLocal', local_file_prefix(src), dst)
       when [:local, :s3]
         s3cmd('put', src, s3b(dst, dir: dir))
       end
@@ -553,7 +557,7 @@ module Masamune
         hadoop_fs('-mv', src, dst)
       when [:hdfs, :local]
         # NOTE: moveToLocal: Option '-moveToLocal' is not implemented yet
-        hadoop_fs('-copyToLocal', src, local_prefix(dst))
+        hadoop_fs('-copyToLocal', src, local_file_prefix(dst))
         hadoop_fs('-rm', src)
       when [:hdfs, :s3]
         copy_file_to_file(src, s3n(dst, dir: dir))
@@ -569,7 +573,7 @@ module Masamune
         FileUtils.mv(src, dst, file_util_args)
         FileUtils.chmod(FILE_MODE, dst, file_util_args)
       when [:local, :hdfs]
-        hadoop_fs('-moveFromLocal', local_prefix(src), dst)
+        hadoop_fs('-moveFromLocal', local_file_prefix(src), dst)
       when [:local, :s3]
         s3cmd('put', src, s3b(dst, dir: dir))
         FileUtils.rm(src, file_util_args)

data/lib/masamune/version.rb

@@ -21,5 +21,5 @@
 #  THE SOFTWARE.
 
 module Masamune
-  VERSION = '0.15.5'
+  VERSION = '0.15.6'
 end

data/spec/masamune/cached_filesystem_spec.rb

@@ -27,8 +27,9 @@ describe Masamune::CachedFilesystem do
   context 'when path is present, top down traversal' do
     before do
       filesystem.touch!('/a/b/c/1.txt', '/a/b/c/2.txt', '/a/b/c/3.txt')
-      expect(filesystem).to receive(:glob_stat).with('/a/b/*').once.and_call_original
-      expect(filesystem).to receive(:glob_stat).with('/a').once.and_call_original
+      expect(filesystem).to receive(:glob_stat).with('/a/b/c/*').once.and_call_original
+      expect(filesystem).to receive(:glob_stat).with('/a/b/*').never
+      expect(filesystem).to receive(:glob_stat).with('/a/*').never
       expect(filesystem).to receive(:glob_stat).with('/*').never
     end
 
@@ -39,7 +40,13 @@ describe Masamune::CachedFilesystem do
       expect(cached_filesystem.exists?('/a/b/c/4.txt')).to eq(false)
       expect(cached_filesystem.exists?('/a/b/c')).to eq(true)
       expect(cached_filesystem.glob('/a/b/c/*').count).to eq(3)
+      expect(cached_filesystem.glob('/a/b/c/*')).to include '/a/b/c/1.txt'
+      expect(cached_filesystem.glob('/a/b/c/*')).to include '/a/b/c/2.txt'
+      expect(cached_filesystem.glob('/a/b/c/*')).to include '/a/b/c/3.txt'
       expect(cached_filesystem.glob('/a/b/c/*.txt').count).to eq(3)
+      expect(cached_filesystem.glob('/a/b/c/*.txt')).to include '/a/b/c/1.txt'
+      expect(cached_filesystem.glob('/a/b/c/*.txt')).to include '/a/b/c/2.txt'
+      expect(cached_filesystem.glob('/a/b/c/*.txt')).to include '/a/b/c/3.txt'
       expect(cached_filesystem.stat('/a/b/c/1.txt')).to_not be_nil
       expect(cached_filesystem.stat('/a/b/c/2.txt')).to_not be_nil
       expect(cached_filesystem.stat('/a/b/c/3.txt')).to_not be_nil
@@ -53,7 +60,8 @@ describe Masamune::CachedFilesystem do
   context 'when path is present, bottom up traversal' do
     before do
       filesystem.touch!('/a/b/c/1.txt', '/a/b/c/2.txt', '/a/b/c/3.txt')
-      expect(filesystem).to receive(:glob_stat).with('/a/*').once.and_call_original
+      expect(filesystem).to receive(:glob_stat).with('/a/b/*').once.and_call_original
+      expect(filesystem).to receive(:glob_stat).with('/a/*').never
       expect(filesystem).to receive(:glob_stat).with('/*').never
     end
 
@@ -78,8 +86,9 @@ describe Masamune::CachedFilesystem do
   context 'when path is present, checking for similar non existant paths' do
     before do
       filesystem.touch!('/y=2013/m=1/d=22/00000')
-      expect(filesystem).to receive(:glob_stat).with('/y=2013/m=1/*').once.and_call_original
-      expect(filesystem).to receive(:glob_stat).with('/y=2013/*').once.and_call_original
+      expect(filesystem).to receive(:glob_stat).with('/y=2013/m=1/d=22/*').once.and_call_original
+      expect(filesystem).to receive(:glob_stat).with('/y=2013/m=1/*').never
+      expect(filesystem).to receive(:glob_stat).with('/y=2013/*').never
       expect(filesystem).to receive(:glob_stat).with('/*').never
     end
 
@@ -87,8 +96,11 @@ describe Masamune::CachedFilesystem do
       expect(cached_filesystem.exists?('/y=2013/m=1/d=22/00000')).to eq(true)
       expect(cached_filesystem.exists?('/y=2013/m=1/d=22')).to eq(true)
       expect(cached_filesystem.exists?('/y=2013/m=1/d=2')).to eq(false)
-      expect(cached_filesystem.glob('/y=2013/m=1/*').count).to eq(1)
+      expect(cached_filesystem.glob('/y=2013/m=1/*').count).to eq(2)
+      expect(cached_filesystem.glob('/y=2013/m=1/*')).to include('/y=2013/m=1/d=22')
+      expect(cached_filesystem.glob('/y=2013/m=1/*')).to include('/y=2013/m=1/d=22/00000')
       expect(cached_filesystem.glob('/y=2013/m=1/d=22/*').count).to eq(1)
+      expect(cached_filesystem.glob('/y=2013/m=1/d=22/*')).to include('/y=2013/m=1/d=22/00000')
       expect(cached_filesystem.stat('/y=2013/m=1/d=22/00000')).not_to be_nil
       expect(cached_filesystem.stat('/y=2013/m=1/d=22')).not_to be_nil
       expect(cached_filesystem.stat('/y=2013/m=1')).not_to be_nil
@@ -116,9 +128,15 @@ describe Masamune::CachedFilesystem do
       expect(cached_filesystem.glob('/logs/*').count).to eq(3)
       expect(cached_filesystem.glob('/logs/*.txt').count).to eq(3)
       expect(cached_filesystem.glob('/logs/box1_*.txt').count).to eq(1)
+      expect(cached_filesystem.glob('/logs/box1_*.txt')).to include('/logs/box1_123.txt')
       expect(cached_filesystem.glob('/logs/box2_*.txt').count).to eq(1)
+      expect(cached_filesystem.glob('/logs/box2_*.txt')).to include('/logs/box2_123.txt')
       expect(cached_filesystem.glob('/logs/box3_*.txt').count).to eq(1)
+      expect(cached_filesystem.glob('/logs/box3_*.txt')).to include('/logs/box3_123.txt')
       expect(cached_filesystem.glob('/logs/box*.txt').count).to eq(3)
+      expect(cached_filesystem.glob('/logs/box*.txt')).to include('/logs/box2_123.txt')
+      expect(cached_filesystem.glob('/logs/box*.txt')).to include('/logs/box1_123.txt')
+      expect(cached_filesystem.glob('/logs/box*.txt')).to include('/logs/box3_123.txt')
       expect(cached_filesystem.glob('/logs/box*.csv').count).to eq(0)
       expect(cached_filesystem.glob('/logs/box').count).to eq(0)
       expect(cached_filesystem.glob('/logs/box/*').count).to eq(0)
@@ -138,7 +156,7 @@ describe Masamune::CachedFilesystem do
     before do
       filesystem.touch!('/a/b/c')
       expect(filesystem).to receive(:glob_stat).with('/a/b/*').once.and_call_original
-      expect(filesystem).to receive(:glob_stat).with('/a').once.and_call_original
+      expect(filesystem).to receive(:glob_stat).with('/a').never
       expect(filesystem).to receive(:glob_stat).with('/*').never
     end
 
@@ -149,11 +167,17 @@ describe Masamune::CachedFilesystem do
       expect(cached_filesystem.exists?('/a/b/c')).to eq(true)
       expect(cached_filesystem.exists?('/a/b')).to eq(true)
       expect(cached_filesystem.exists?('/a')).to eq(true)
-      expect(cached_filesystem.glob('/a').count).to eq(0)
-      expect(cached_filesystem.glob('/a/*').count).to eq(1)
-      expect(cached_filesystem.glob('/a/b').count).to eq(0)
+      expect(cached_filesystem.glob('/a').count).to eq(1)
+      expect(cached_filesystem.glob('/a')).to include '/a'
+      expect(cached_filesystem.glob('/a/*').count).to eq(2)
+      expect(cached_filesystem.glob('/a/*')).to include '/a/b'
+      expect(cached_filesystem.glob('/a/*')).to include '/a/b/c'
+      expect(cached_filesystem.glob('/a/b').count).to eq(1)
+      expect(cached_filesystem.glob('/a/b')).to include '/a/b'
       expect(cached_filesystem.glob('/a/b/*').count).to eq(1)
+      expect(cached_filesystem.glob('/a/b/*')).to include '/a/b/c'
       expect(cached_filesystem.glob('/a/b/c').count).to eq(1)
+      expect(cached_filesystem.glob('/a/b/c')).to include '/a/b/c'
       expect(cached_filesystem.glob('/a/b/c/*').count).to eq(0)
       expect(cached_filesystem.glob('/a/b/c/*.txt').count).to eq(0)
       expect(cached_filesystem.stat('/a/b/c/1.txt')).to be_nil
@@ -164,4 +188,118 @@ describe Masamune::CachedFilesystem do
       expect(cached_filesystem.stat('/a')).to_not be_nil
     end
   end
+
+  describe '#glob_stat' do
+    before do
+      filesystem.touch!('/a/b/c/1.txt', '/a/b/c/2.txt', '/a/b/c/3.txt')
+    end
+
+    context 'without options' do
+      before do
+        expect(filesystem).to receive(:glob_stat).with('/a/b/c/*').once.and_call_original
+        expect(filesystem).to receive(:glob_stat).with('/a/b/*').never
+        expect(filesystem).to receive(:glob_stat).with('/a/*').never
+        expect(filesystem).to receive(:glob_stat).with('/*').never
+      end
+      it do
+        expect(cached_filesystem.glob_stat('/a/b/c/1.txt').count).to eq(1)
+      end
+    end
+
+    context 'with max_depth=1' do
+      before do
+        expect(filesystem).to receive(:glob_stat).with('/a/b/c/*').never
+        expect(filesystem).to receive(:glob_stat).with('/a/b/*').once.and_call_original
+        expect(filesystem).to receive(:glob_stat).with('/a/*').never
+        expect(filesystem).to receive(:glob_stat).with('/*').never
+      end
+      it { expect(cached_filesystem.glob_stat('/a/b/c/1.txt', max_depth: 1).count).to eq(1) }
+    end
+
+    context 'with max_depth=2' do
+      before do
+        expect(filesystem).to receive(:glob_stat).with('/a/b/c/*').never
+        expect(filesystem).to receive(:glob_stat).with('/a/b/*').never
+        expect(filesystem).to receive(:glob_stat).with('/a/*').once.and_call_original
+        expect(filesystem).to receive(:glob_stat).with('/*').never
+      end
+      it { expect(cached_filesystem.glob_stat('/a/b/c/1.txt', max_depth: 2).count).to eq(1) }
+    end
+
+    context 'with max_depth=3' do
+      before do
+        expect(filesystem).to receive(:glob_stat).with('/a/b/c/*').never
+        expect(filesystem).to receive(:glob_stat).with('/a/b/*').never
+        expect(filesystem).to receive(:glob_stat).with('/a/*').once.and_call_original
+        expect(filesystem).to receive(:glob_stat).with('/*').never
+      end
+      it { expect(cached_filesystem.glob_stat('/a/b/c/1.txt', max_depth: 3).count).to eq(1) }
+    end
+  end
+
+  describe Masamune::CachedFilesystem::PathCache do
+    let(:instance) { described_class.new(filesystem) }
+
+    before do
+      instance.put('/a/b/c/1.txt', OpenStruct.new(name: '/a/b/c/1.txt'))
+      instance.put('/a/b/c/2.txt', OpenStruct.new(name: '/a/b/c/2.txt'))
+      instance.put('/a/b/c/3.txt', OpenStruct.new(name: '/a/b/c/3.txt'))
+    end
+
+    subject(:result) do
+      instance.get(path)
+    end
+
+    context 'with nil' do
+      let(:path) { nil }
+
+      it { is_expected.to be_nil }
+    end
+
+    context 'with existing file path' do
+      let(:path) { '/a/b/c/1.txt' }
+
+      it 'returns existing file' do
+        expect(result).to include(OpenStruct.new(name: '/a/b/c/1.txt'))
+        expect(result.count).to eq(1)
+      end
+    end
+
+    context 'with existing directory path' do
+      let(:path) { '/a/b/c' }
+
+      it 'returns matching files' do
+        expect(result).to include(OpenStruct.new(name: '/a/b/c/1.txt'))
+        expect(result).to include(OpenStruct.new(name: '/a/b/c/2.txt'))
+        expect(result).to include(OpenStruct.new(name: '/a/b/c/3.txt'))
+        expect(result).to include(OpenStruct.new(name: '/a/b/c'))
+        expect(result.count).to eq(4)
+      end
+    end
+
+    context 'with existing directory path (nested)' do
+      let(:path) { '/a/b' }
+
+      it 'returns matching files' do
+        expect(result).to include(OpenStruct.new(name: '/a/b/c/1.txt'))
+        expect(result).to include(OpenStruct.new(name: '/a/b/c/2.txt'))
+        expect(result).to include(OpenStruct.new(name: '/a/b/c/3.txt'))
+        expect(result).to include(OpenStruct.new(name: '/a/b/c'))
+        expect(result).to include(OpenStruct.new(name: '/a/b'))
+        expect(result.count).to eq(5)
+      end
+    end
+
+    context 'with missing file path' do
+      let(:path) { '/a/b/c/4.txt' }
+
+      it { is_expected.to be_empty }
+    end
+
+    context 'with missing directory path' do
+      let(:path) { '/a/b/d' }
+
+      it { is_expected.to be_empty }
+    end
+  end
 end

data/spec/masamune/filesystem_spec.rb

@@ -212,7 +212,7 @@ shared_examples_for 'Filesystem' do
 
     context 'with local blank' do
       let(:path) { '' }
-      it { is_expected.to be_blank }
+      it { is_expected.to eq('.') }
     end
 
     context 'with local path with slash' do
@@ -220,6 +220,11 @@ shared_examples_for 'Filesystem' do
       it { is_expected.to eq('/a/b') }
     end
 
+    context 'with local file without slash' do
+      let(:path) { 'a' }
+      it { is_expected.to eq('.') }
+    end
+
     context 'with local path without slash' do
       let(:path) { 'a/b/c' }
       it { is_expected.to eq('a/b') }
@@ -227,7 +232,12 @@ shared_examples_for 'Filesystem' do
 
     context 'with local relative path' do
       let(:path) { '/a/b/../c' }
-      it { is_expected.to eq('/a/c') }
+      it { is_expected.to eq('/a') }
+    end
+
+    context 'with local another relative path' do
+      let(:path) { '/a/b/.' }
+      it { is_expected.to eq('/a') }
     end
 
     context 'with s3 bucket with blank' do
@@ -237,7 +247,7 @@ shared_examples_for 'Filesystem' do
 
     context 'with s3 bucket with slash' do
       let(:path) { 's3://bucket/' }
-      it { is_expected.to eq('s3://bucket/') }
+      it { is_expected.to eq('s3://bucket') }
     end
 
     context 'with s3 bucket with path' do
@@ -247,7 +257,7 @@ shared_examples_for 'Filesystem' do
 
     context 'with s3 bucket with relative path' do
       let(:path) { 's3://bucket/a/b/../c' }
-      it { is_expected.to eq('s3://bucket/a/c') }
+      it { is_expected.to eq('s3://bucket/a') }
     end
 
     context 'with hdfs directory with path' do
@@ -262,7 +272,7 @@ shared_examples_for 'Filesystem' do
 
     context 'with hdfs directory with relative path' do
       let(:path) { 'hdfs:///a/b/../c' }
-      it { is_expected.to eq('hdfs:///a/c') }
+      it { is_expected.to eq('hdfs:///a') }
     end
   end
 
@@ -362,7 +372,7 @@ shared_examples_for 'Filesystem' do
 
     context 'hdfs missing file' do
       before do
-        expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once).
+        expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once).
           and_yield("drwxrwxrwt   - root     wheel         68 2015-02-24 12:09 #{old_dir}")
         expect(filesystem).to receive(:hadoop_fs).with('-test', '-e', 'file://' + new_file, safe: true).at_most(:once).and_return(mock_failure)
       end
@@ -385,7 +395,7 @@ shared_examples_for 'Filesystem' do
 
     context 'hdfs existing file' do
       before do
-        expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once).
+        expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once).
           and_yield("drwxrwxrwt   - root     wheel         68 2015-02-24 12:09 #{old_dir}").
           and_yield("drwxrwxrwt   - root     wheel         68 2015-02-24 12:09 #{old_file}")
         expect(filesystem).to receive(:hadoop_fs).with('-test', '-e', 'file://' + old_file, safe: true).at_most(:once).and_return(mock_success)
@@ -434,7 +444,7 @@ shared_examples_for 'Filesystem' do
 
     context 'hdfs missing file' do
       before do
-        expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once).
+        expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once).
           and_yield('')
         expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once).
           and_yield('')
@@ -489,7 +499,7 @@ shared_examples_for 'Filesystem' do
 
     context 'hdfs existing file' do
       before do
-        expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once).
+        expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once).
           and_yield("drwxrwxrwt   - root     wheel         68 2015-02-24 12:09 #{old_dir}").
           and_yield("drwxrwxrwt   - root     wheel         68 2015-02-24 12:09 #{old_file}")
         expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_file, safe: true).at_most(:once).
@@ -549,7 +559,7 @@ shared_examples_for 'Filesystem' do
 
       describe '#name' do
         subject { stat.name }
-        it { is_expected.to eq('s3://bucket/dir/file.txt') }
+        it { is_expected.to eq('s3://bucket/dir') }
       end
 
       describe '#mtime' do

data/spec/masamune/rspec/mock_filesystem_spec.rb

@@ -0,0 +1,129 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+describe Masamune::MockFilesystem do
+  let(:instance) { described_class.new }
+
+  describe '#glob' do
+    before do
+      instance.touch!('/a/b/c/1.txt', '/a/b/c/2.txt', '/a/b/c/3.txt')
+    end
+
+    subject(:result) { instance.glob(input) }
+
+    context 'with glob for existing file' do
+      let(:input) { '/a/b/c/1.txt' }
+
+      it 'contains single matching file' do
+        expect(result).to include('/a/b/c/1.txt')
+        expect(result.count).to eq(1)
+      end
+    end
+
+    context 'with glob for existing files' do
+      let(:input) { '/a/b/c/*' }
+
+      it 'contains all matching files' do
+        expect(result).to include('/a/b/c/1.txt')
+        expect(result).to include('/a/b/c/2.txt')
+        expect(result).to include('/a/b/c/3.txt')
+        expect(result.count).to eq(3)
+      end
+    end
+
+    context 'with glob for existing files (recursive)' do
+      let(:input) { '/a/b/*' }
+
+      it 'contains all matching files and directory' do
+        expect(result).to include('/a/b/c')
+        expect(result).to include('/a/b/c/1.txt')
+        expect(result).to include('/a/b/c/2.txt')
+        expect(result).to include('/a/b/c/3.txt')
+        expect(result.count).to eq(4)
+      end
+    end
+
+    context 'with glob for missing file' do
+      let(:input) { '/a/b/c/4.txt' }
+
+      it { expect(result).to be_empty }
+    end
+
+    context 'with glob for missing directory' do
+      let(:input) { '/a/b/d/*' }
+
+      it { expect(result).to be_empty }
+    end
+  end
+
+  describe '#glob_stat' do
+    before do
+      instance.touch!('/a/b/c/1.txt', '/a/b/c/2.txt', '/a/b/c/3.txt')
+    end
+
+    subject(:result) { instance.glob_stat(input).map(&:name) }
+
+    context 'with glob for existing file' do
+      let(:input) { '/a/b/c/1.txt' }
+
+      it 'contains single matching file' do
+        expect(result).to include('/a/b/c/1.txt')
+        expect(result.count).to eq(1)
+      end
+    end
+
+    context 'with glob for existing files' do
+      let(:input) { '/a/b/c/*' }
+
+      it 'contains all matching files' do
+        expect(result).to include('/a/b/c/1.txt')
+        expect(result).to include('/a/b/c/2.txt')
+        expect(result).to include('/a/b/c/3.txt')
+        expect(result.count).to eq(3)
+      end
+    end
+
+    context 'with glob for existing files (recursive)' do
+      let(:input) { '/a/b/*' }
+
+      it 'contains all matching files and directory' do
+        expect(result).to include('/a/b/c')
+        expect(result).to include('/a/b/c/1.txt')
+        expect(result).to include('/a/b/c/2.txt')
+        expect(result).to include('/a/b/c/3.txt')
+        expect(result.count).to eq(4)
+      end
+    end
+
+    context 'with glob for missing file' do
+      let(:input) { '/a/b/c/4.txt' }
+
+      it { expect(result).to be_empty }
+    end
+
+    context 'with glob for missing directory' do
+      let(:input) { '/a/b/d/*' }
+
+      it { expect(result).to be_empty }
+    end
+  end
+end

data/spec/support/masamune/mock_filesystem.rb

@@ -32,19 +32,24 @@ class Masamune::MockFilesystem < Delegator
   def touch!(*args)
     opts = args.last.is_a?(Hash) ? args.pop : {}
     args.each do |file|
+      parent_paths(file).each do |parent|
+        @files[parent] = OpenStruct.new(opts.merge(name: parent))
+      end
       @files[file] = OpenStruct.new(opts.merge(name: file))
     end
   end
 
   def exists?(file)
-    @files.keys.include?(file)
+    @files.keys.any? { |path| file == path || path.start_with?(File.join(file, '/')) }
   end
 
-  def glob(pattern)
-    return Set.new(to_enum(:glob, pattern)) unless block_given?
+  def glob(pattern, options = {})
+    return Set.new(to_enum(:glob, pattern, options)) unless block_given?
     file_regexp = glob_to_regexp(pattern)
     @files.keys.each do |name|
-      yield name if name =~ file_regexp
+      next if name == dirname(pattern)
+      next unless name =~ file_regexp
+      yield name
     end
   end
 
@@ -55,11 +60,13 @@ class Masamune::MockFilesystem < Delegator
     end
   end
 
-  def glob_stat(pattern)
-    return Set.new(to_enum(:glob_stat, pattern)) unless block_given?
+  def glob_stat(pattern, options = {})
+    return Set.new(to_enum(:glob_stat, pattern, options)) unless block_given?
     file_regexp = glob_to_regexp(pattern, recursive: true)
     @files.each do |name, stat|
-      yield stat if name =~ file_regexp
+      next if stat.name == dirname(pattern)
+      next unless stat.name =~ file_regexp
+      yield stat
     end
   end
 

data/spec/support/masamune/shared_example_group.rb

@@ -118,7 +118,6 @@ module Masamune::SharedExampleGroup
         expect_data = expect_data.split("\n").sort.join("\n")
       end
       yield [actual_data, output_file, expect_data]
-      FileUtils.rm(output_file) if File.exists?(output_file)
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: masamune
 version: !ruby/object:Gem::Version
-  version: 0.15.5
+  version: 0.15.6
 platform: ruby
 authors:
 - Michael Andrews
@@ -365,6 +365,7 @@ files:
 - spec/masamune/filesystem_spec.rb
 - spec/masamune/helpers/postgres_spec.rb
 - spec/masamune/rspec/job_fixture_spec.rb
+- spec/masamune/rspec/mock_filesystem_spec.rb
 - spec/masamune/rspec/shared_example_group_spec.rb
 - spec/masamune/schema/catalog_spec.rb
 - spec/masamune/schema/column_spec.rb
@@ -476,6 +477,7 @@ test_files:
 - spec/masamune/filesystem_spec.rb
 - spec/masamune/helpers/postgres_spec.rb
 - spec/masamune/rspec/job_fixture_spec.rb
+- spec/masamune/rspec/mock_filesystem_spec.rb
 - spec/masamune/rspec/shared_example_group_spec.rb
 - spec/masamune/schema/catalog_spec.rb
 - spec/masamune/schema/column_spec.rb