vfs 0.0.4 → 0.1.0

data/lib/vfs/storages/specification.rb

@@ -0,0 +1,127 @@
+# use '$ gem install ruby_ext' to install.
+require 'rspec_ext'
+require 'ruby_ext'
+
+shared_examples_for 'vfs storage' do  
+  before :each do
+    @storage.open_fs do |fs|
+      @tmp_dir = fs.tmp
+    end
+  end
+
+  after :each do
+    @storage.open_fs do |fs|
+      attrs = fs.attributes(@tmp_dir)
+      fs.delete_dir @tmp_dir if attrs && attrs[:dir]
+    end
+  end
+  
+  it 'should respond to :local?' do
+    @storage.open_fs{|fs| fs.should respond_to(:local?)}
+  end
+  
+  it 'should have root dir' do
+    @storage.open_fs do |fs|
+      fs.attributes('/').subset(:file, :dir).should == {file: false, dir: true}
+    end
+  end
+    
+  describe "files" do  
+    before :each do
+      @remote_file = "#{@tmp_dir}/remote_file"
+    end
+    
+    it "file attributes" do
+      @storage.open_fs do |fs|
+        fs.attributes(@remote_file).should == {}
+        fs.write_file(@remote_file, false){|w| w.call 'something'}
+        attrs = fs.attributes(@remote_file)      
+        fs.attributes(@remote_file).subset(:file, :dir).should == {file: true, dir: false}
+      end
+    end
+
+    it "read, write & append" do
+      @storage.open_fs do |fs|
+        fs.write_file(@remote_file, false){|w| w.call 'something'}
+        fs.attributes(@remote_file)[:file].should be_true
+      
+        data = ""  
+        fs.read_file(@remote_file){|buff| data << buff}
+        data.should == 'something'
+      
+        # append
+        fs.write_file(@remote_file, true){|w| w.call ' another'}
+        data = ""  
+        fs.read_file(@remote_file){|buff| data << buff}
+        data.should == 'something another'
+      end
+    end
+  
+    it "delete_file" do
+      @storage.open_fs do |fs|
+        fs.write_file(@remote_file, false){|w| w.call 'something'}        
+        fs.attributes(@remote_file)[:file].should be_true
+        fs.delete_file(@remote_file)
+        fs.attributes(@remote_file).should == {}
+      end
+    end
+  end
+  
+  describe 'directories' do
+    # before :each do
+    #   @from_local, @remote_path, @to_local = "#{@local_dir}/dir", "#{@tmp_dir}/upload", "#{@local_dir}/download"
+    # end
+    
+    before :each do
+      @remote_dir = "#{@tmp_dir}/some_dir"
+    end
+    
+    it "directory_exist?, create_dir, delete_dir" do        
+      @storage.open_fs do |fs|
+        fs.attributes(@remote_dir).should == {}
+        fs.create_dir(@remote_dir)
+        fs.attributes(@remote_dir).subset(:file, :dir).should == {file: false, dir: true}
+        fs.delete_dir(@remote_dir)
+        fs.attributes(@remote_dir).should == {}
+      end
+    end
+    
+    it 'should delete not-empty directories' do
+      @storage.open_fs do |fs|
+        fs.create_dir(@remote_dir)
+        fs.create_dir("#{@remote_dir}/dir")
+        fs.write_file("#{@remote_dir}/dir/file", false){|w| w.call 'something'}         
+        fs.delete_dir(@remote_dir)
+        fs.attributes(@remote_dir).should == {}
+      end
+    end
+    
+    it 'each' do
+      @storage.open_fs do |fs|
+        list = {}
+        fs.each(@tmp_dir){|path, type| list[path] = type}
+        list.should be_empty
+      
+        dir, file = "#{@tmp_dir}/dir", "#{@tmp_dir}/file"
+        fs.create_dir(dir)
+        fs.write_file(file, false){|w| w.call 'something'}
+      
+        list = {}
+        fs.each(@tmp_dir){|path, type| list[path] = type}
+        list.should == {'dir' => :dir, 'file' => :file}
+      end
+    end
+  
+    # it "upload_directory & download_directory" do
+    #   upload_path_check = "#{@remote_path}/dir2/file"
+    #   check_attributes upload_path_check, nil        
+    #   fs.upload_directory(@from_local, @remote_path)
+    #   check_attributes upload_path_check, file: true, dir: false
+    #   
+    #   download_path_check = "#{@to_local}/dir2/file"
+    #   File.exist?(download_path_check).should be_false
+    #   fs.download_directory(@remote_path, @to_local)
+    #   File.exist?(download_path_check).should be_true
+    # end
+  end  
+end

data/lib/vfs/support.rb

@@ -0,0 +1,2 @@
+require 'fileutils'
+require 'set'

data/readme.md

@@ -1,21 +1,117 @@
-# Rsh - Tiny wrapper over Net::SSH/SFTP
+# Vfs - Virtual File System
 
-Because they are too hard to use and have terrible API design.
+Handy and simple abstraction over any storage that can represent concept of File and Directory (or at least part of it). 
+The Vfs for File System Storages is the same as ActiveRecord is for Relational Databases.
 
-    box = Rsh::Box.new host: 'webapp.com', ssh: {user: 'root', password: 'secret'}
+Currently, there are following implementations available:
 
-    box.upload_directory '/my_project', '/apps/my_project'
-    box.bash 'nohup /apps/my_project/server_start'
-  
-Honestly my wrapper also not very good. I would like to make API looks like the ['rush'][rush] gem (made by Adam Wiggins)
-but it requires a lots of time, maybe I'll do it later.
-So, for now it's just a small wrapper to do ssh/io operations not so painfull.
+- local file system
+- remote file system (over ssh)
+
+## Goals
+
+- **handy, simple and clean** API.
+- **high performance** - the same as by using low-level storage API, there should be no extra calls **.
+- same API for different storages (Local FS, SSH, Hadoop, or any other , ...).
+- should work **simultaneously with different storages**.
+- small codebase, easy to extend by others.
+- simple storage-driver implementation, easy add new storage types (Hadoop DFS, LDAP, Document Oriented DB, In-Memory, ...).
+
+** all methods should have the same performance as native system calls, except for :move and :rename. Right now they are implemented 
+ASAP by using copy+destroy approach, will be fixed as soon as I'll have free time to do it.
+
+## Code samples:
+      gem 'vfs'                                    # Virtual File System
+      require 'vfs'                              
+
+      gem 'vos'                                    # Virtual Operating System
+      require 'vos'
+
+
+      # Connections, let's deploy our 'cool_app' project from our local box to remote server
+      server = Vfs::Box.new(host: 'cool_app.com', ssh: {user: 'me', password: 'secret'})
+      me = '~'.to_dir
+
+      cool_app = server['apps/cool_app']
+      projects = me['projects']
+
+
+      # Working with dirs, copying dir from any source to any destination (local/remote/custom_storage_type)
+      projects['cool_app'].copy_to cool_app        
+
+
+      # Working with files
+      dbc = cool_app.file('config/database.yml')   # <= the 'config' dir not exist yet
+      dbc.write("user: root\npassword: secret")    # <= now the 'database.yml' and parent 'config' has been created
+      dbc.content =~ /database/                    # => false, we forgot to add the database
+      dbc.append("\ndatabase: mysql")              # let's do it
+
+      dbc.update do |content|                      # and add host info
+        content + "\nhost: cool_app.com "
+      end                                       
+
+      projects['cool_app/config/database.yml'].    # or just overwrite it with our local dev version
+        copy_to! dbc
+        
+      # there are also streaming support (read/write/append), please go to specs for docs
+
+
+      # Checks
+      cool_app['config'].exist?                    # => true
+      cool_app.dir('config').exist?                # => true
+      cool_app.file('config').exist?               # => false
 
+      cool_app['config'].dir?                      # => true
+      cool_app['config'].file?                     # => false
+
+
+      # Navigation
+      config = cool_app['config']
+      config.parent                                # => </apps/cool_app>
+      config['../..']                              # => </>
+      config['../..'].dir?                         # => true
+
+      cool_app.entries                             # => list of dirs and files, also support &block
+      cool_app.files                               # => list of files, also support &block
+      cool_app.dirs                                # => list of dirs, also support &block
+
+
+      # For more please go to specs (create/update/move/copy/destroy/...)
+      
+## Integration with [Vos][vos] (Virtual Operating System)
+    
+      server['apps/cool_app'].bash 'rails production'
+
+For more details please go to [Vos][vos] project page.
+
+# Why?
+
+To easy my work: with local FS, remote FS (cluster management, deployment automation), and some specific systems like Hadoop DFS.
+
+Because the API of standard File/Dir/FileUtils classes are just terrible. And there's the reason for it - the goal of thouse tools
+is to provide 1-to-1 clone of underlying OS API, instead of provididing handy tool.
+
+And if you want to use remote FS - things are getting even worse and more complicated (Net::SSH & Net::SFTP use a little
+different API than local FS, and you has to remember all thouse little quirks).
+  
 ## TODO
 
-- remove ssh.remote and use only open/close
-- introduce Entity/Dir/File (the same as in Rush)
-- allow to move files between any Boxes, not only between local and remote.
-- add support for moving dirs.
+### v 0.1 (all done)
+
+- Vos: Dir.bash
+- File.append
+- list of entries/files/dirs
+- support for efficient copy for Local and SSH storages
+
+### v 0.2 (not started)
+
+- efficient (not copy/destroy) versions of move_to, rename
+- glob search for directories: Dir['**/*.yml']
+- access via attributes and helpers for unix chmod
+- add storages: remote FS over HTTP.
+
+### future
+
+- add storages: Hadoop DFS, MongoDB, Amazon S3
 
-[rush]: http://github.com/adamwiggins/rush
+[vos]: http://github.com/alexeypetrushin/vos

data/spec/container_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe 'Container' do
+  before :each do
+    @fs = '/'.to_entry_on(Vfs::Storages::HashFs.new)
+  end
+  
+  it "should threat paths as UniversalEntry except it ends with '/'" do
+    @fs.should_receive(:entry).with('/a/b')
+    @fs['/a/b']
+    
+    @fs.should_receive(:dir).with('/a/b')
+    @fs['/a/b/']
+  end
+  
+  it '/' do
+    @fs[:some_path].should == @fs / :some_path
+  end
+  
+  it "UniversalEntry should be wrapped inside of proxy, Dir and File should not" do
+    -> {@fs.dir.proxy?}.should raise_error(NoMethodError)
+    -> {@fs.file.proxy?}.should raise_error(NoMethodError)
+    @fs.entry.proxy?.should be_true
+  end
+  
+  it "sometimes it also should inexplicitly guess that path is a Dir instead of UniversalEntry (but still wrap it inside of Proxy)" do
+    dir = @fs['/a/..']
+    dir.proxy?.should be_true
+    dir.should be_a(Vfs::Dir)
+  end
+end

data/spec/dir_spec.rb

@@ -0,0 +1,224 @@
+require 'spec_helper'
+
+describe 'Dir' do
+  before :each do
+    @fs = '/'.to_entry_on(Vfs::Storages::HashFs.new)
+    @path = @fs['/a/b/c']
+  end
+  
+  describe 'existence' do
+    it "should check only dirs" do
+      @path.should_not exist
+      @path.file.create
+      @path.should be_file
+      @path.dir.should_not exist
+      @path.dir.create!
+      @path.should be_dir
+      @path.dir.should exist
+    end
+  end
+    
+  it "should not respond to read and write methods" do
+    -> {@path.dir.read}.should raise_error(NoMethodError)
+    -> {@path.dir.write}.should raise_error(NoMethodError)
+  end
+  
+  describe 'create' do
+    it 'should be chainable' do
+      @path.dir.create.should == @path
+      @path.dir.create!.should == @path
+    end
+    
+    it 'should create parent dirs if not exists' do
+      @path.parent.should_not exist
+      @path.dir.create
+      @path.should be_dir
+    end
+    
+    it 'should silently exit if dir already exist' do
+      @path.dir.create
+      @path.dir.create
+    end
+    
+    it 'should override existing file if override specified' do
+      @path.file.create
+      @path.should be_file
+      -> {@path.dir.create}.should raise_error(Vfs::Error, /exist/)
+      @path.dir.create!
+      @path.should be_dir
+    end
+    
+    it 'should override existing dir if override specified' do
+      @path.dir.create
+      @path.should be_dir
+      @path.dir.create!
+      @path.should be_dir
+    end
+  end
+  
+  describe 'destroying' do
+    it "should raise error if it's trying to destroy a file (unless force specified)" do
+      @path.file.create
+      -> {@path.dir.destroy}.should raise_error(Vfs::Error, /can't destroy File/)
+      @path.dir.destroy!
+      @path.entry.should_not exist
+    end
+    
+    it "shouldn't raise if dir not exist" do
+      @path.dir.destroy
+    end
+    
+    it 'should destroy recursivelly' do
+      dir = @path.dir
+      dir.create
+      dir.file('file').write 'something'
+      dir.dir('dir').create.tap do |dir|
+        dir.file('file2').write 'something2'        
+      end
+      
+      dir.destroy
+      dir.should_not exist
+    end
+    
+    it 'should be chainable' do
+      @path.dir.destroy.should == @path
+      @path.dir.destroy!.should == @path
+    end
+  end
+  
+  describe 'content' do
+    before :each do
+      @path.dir('dir').create
+      @path.dir('dir/another_dir').create
+      @path.file('file').create      
+    end
+    
+    it 'entries' do
+      -> {@path['non_existing'].entries}.should raise_error(Vfs::Error, /not exist/)
+      @path['non_existing'].entries(bang: false).should == []
+      @path.entries.to_set.should be_eql([@path.dir('dir'), @path.file('file')].to_set)
+      list = []
+      @path.entries{|e| list << e}
+      list.to_set.should be_eql([@path.dir('dir'), @path.file('file')].to_set)
+    end
+    
+    it 'should raise error if trying :entries on file' do
+      @path.file('some_file').create
+      -> {@path.dir('some_file').entries}.should raise_error(/File/)
+    end
+    
+    it 'files' do
+      @path.files.should be_eql([@path.file('file')])
+    end
+    
+    it 'dirs' do
+      @path.dirs.should be_eql([@path.dir('dir')])
+    end
+    
+    it 'has? & include?' do
+      @path.include?('dir').should be_true
+      @path.include?('dir/another_dir').should be_true
+      @path.include?('file').should be_true
+      @path.include?('non_existing').should be_false
+    end
+  end
+  
+  describe 'copying' do
+    before :each do 
+      @from = @path.dir
+      @from.create
+      @from.file('file').write 'something'
+      @from.dir('dir').create.tap do |dir|
+        dir.file('file2').write 'something2'        
+      end
+    end
+    
+    it 'should not copy to itself' do
+      -> {@from.copy_to @from}.should raise_error(Vfs::Error, /itself/)
+    end
+    
+    def check_copy_for to, error_re1, error_re2 
+      begin
+        target = @from.copy_to to                  
+        target['file'].read.should == 'something'
+        target['dir/file2'].read.should == 'something2'
+        target.should == to
+      rescue Exception => e
+        raise e unless e.message =~ error_re1
+      end      
+      
+      @from['dir/file2'].write! 'another'      
+      -> {@from.copy_to to}.should raise_error(Vfs::Error, error_re2)
+      target = @from.copy_to! to
+      target['file'].read.should == 'something'
+      target['dir/file2'].read.should == 'another'            
+    end
+    
+    describe 'general copy' do   
+      before :each do
+        # we using here another HashFs storage, to prevent :effective_dir_copy to be used
+        @to = '/'.to_entry_on(Vfs::Storages::HashFs.new)['to']
+        
+        @from.storage.should_not_receive(:for_spec_helper_effective_copy_used)
+      end      
+    
+      it 'should copy to file (and overwrite if forced)' do
+        check_copy_for @to.file, /can't copy Dir to File/, /can't copy Dir to File/
+      end
+    
+      it 'should copy to dir (and overwrite if forced)' do
+        check_copy_for @to.dir, nil, /exist/
+      end
+    
+      it 'should copy to UniversalEntry (and overwrite if forced)' do
+        check_copy_for @to.entry, nil, /exist/
+      end
+    end
+    
+    describe 'effective copy' do
+      before :each do
+        # we using the same HashFs storage, so :effective_dir_copy will be used
+        @to = @fs['to']
+        
+        @from.storage.should_receive(:for_spec_helper_effective_copy_used).at_least(1).times
+      end
+      
+      it 'should copy to file (and overwrite if forced)' do
+        check_copy_for @to.file, /can't copy Dir to File/, /can't copy Dir to File/
+      end
+    
+      it 'should copy to dir (and overwrite if forced)' do
+        check_copy_for @to.dir, nil, /exist/
+      end
+    
+      it 'should copy to UniversalEntry (and overwrite if forced)' do
+        check_copy_for @to.entry, nil, /exist/
+      end
+    end
+    
+    it 'should be chainable' do
+      to = @fs['to']
+      @from.copy_to(to).should == to
+      @from.copy_to!(to).should == to
+    end
+  end
+    
+  describe 'moving' do
+    it 'move_to' do
+      from, to = @path.file('from'), @path.file('to')
+      from.should_receive(:copy_to).with(to, {})
+      from.should_receive(:destroy).with({})
+      from.move_to to
+    
+      from.should_receive(:move_to).with(to, override: true)
+      from.move_to! to
+    end
+    
+    it 'should be chainable' do      
+      from, to = @path.dir('from').create, @path.dir('to')
+      from.move_to(to).should == to
+      from.create
+      from.move_to!(to).should == to
+    end
+  end
+end