fmanager 0.0.0 → 0.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c7bf21743d1350607586cba4561d2806c12788e8
-  data.tar.gz: 176f478bd91f3c2c2ab2952069bc60a81f971aca
+  metadata.gz: 5e243865920465668fdca0f5965ee8d6be9b961f
+  data.tar.gz: 7dff3316ad7ee02845121fef9d024d3972e82cab
 SHA512:
-  metadata.gz: e32d737ec16e2b3b45840b755ef680a8b5fbfe897b5da8db62671c77d9e6474799e1600762505356e81672c31c62d23a4849f2b2e9d6537ed1592fc48378e433
-  data.tar.gz: 31ea5db5a193191ac9cfdc577b00c90edb34123a2d0ea07fdb7b197bae9b92c759f3533c2f6f4852591dbf4782258e6e610a0d54e4c0ef1946955ae27be3e135
+  metadata.gz: 50fd935098b10669bd6fa9982891eb423dff60063b72358887e9253dd063febbebf41471f3a7db37cf56c1611873ed3a9bb65832002b78811679bd0dba232d99
+  data.tar.gz: 3981c7b35cfadf325c46ab2370ec0bf84568fd29cdf370d37ae237a3ace4191b35e3505e24032547515b3c83dc1f3c7a37a3ca0d9ecc88dd4b3bfb1ec49a7e2b

data/Rakefile

@@ -1,3 +1,4 @@
+$:.unshift File.expand_path("../lib", __FILE__)
 require 'rake/testtask'
 
 require 'fmanager'

data/bin/fm

@@ -3,4 +3,28 @@
 require 'fmanager'
 
 
-puts "fmanager #{FManager::VERSION}"
+def usage
+    <<EOS
+SGC File Manager #{FManager::VERSION}
+
+usage: fm <command> [args]
+
+Available fm commands:
+    index       Index files in a folder recursively.
+EOS
+end
+
+
+if ARGV.size == 0
+    $stderr << usage
+    exit
+end
+
+cmd = ARGV.shift
+case cmd
+when "index"
+    FManager.cmd_index(ARGV.clone)
+else
+    $stderr << "fm: '#{cmd}' is not a fm command.\n"
+    exit
+end

data/history.txt

@@ -1,7 +1,7 @@
-## 0.0.1 (???)
+## 0.0.1 (18 Aug 2013)
 
 Features:
 
-    * Indexing command with file metadata in YAML database.
+    * Indexing command 'fm index' with file metadata stored in YAML format.
     * Ruby gem package.
     * Support Linux platform.

data/lib/fmanager.rb

@@ -1 +1,2 @@
 require 'fmanager/version'
+require 'fmanager/findex'

data/lib/fmanager/findex.rb

@@ -0,0 +1,142 @@
+require 'digest/md5'
+require 'optparse'
+require 'yaml'
+
+require 'fmanager/fmatch'
+require 'fmanager/fmeta'
+
+
+DBFILE = "fmetadata.db"
+
+
+module FManager
+
+    def cmd_index(argv)
+
+        banner = "usage: fm index [options] <PATH>"
+        options = {
+            dbfile: DBFILE,
+            updatedb: true,
+        }
+        OptionParser.new do |opts|
+            opts.banner = banner
+
+            opts.on "-d", "--db-file PATH", "Specify the DB PATH to use." do |path|
+                options[:dbfile] = path
+            end
+
+            opts.on "-n", "--no-update-db", "Do not update the DB." do
+                options[:updatedb] = false
+            end
+
+            opts.on "-u", "--update-db", "Update the DB when applicable." do
+                options[:updatedb] = true
+            end
+
+            opts.on "-h", "--help", "Show this message." do
+                puts opts
+                exit
+            end
+        end.parse!(argv)
+
+        if argv.size != 1
+            $stderr << "#{banner}\n"
+            exit
+        end
+
+
+        puts "Loading DB #{options[:dbfile]} ..."
+        t0 = Time.now
+        h = {}
+        needupdate = true
+        if File.exists?(options[:dbfile])
+            obj = YAML::load(File.read(options[:dbfile]))
+            if obj.is_a?(Hash)
+                h = obj
+                needupdate = false
+            end
+        end
+        t1 = Time.now
+        puts "Loaded DB in #{(t1-t0).round(2)}s."
+
+        indexpath = argv[0]
+        nnewfiles = 0
+        nskipfiles = 0
+        ndupfiles = 0
+        nfailfiles = 0
+        puts "Indexing #{indexpath} ..."
+        t0 = Time.now
+
+        for fpath in Dir.glob("#{indexpath}/**/*", File::FNM_DOTMATCH).select { |e| e.force_encoding("binary"); File.ftype(e) == "file" && has_folder?(".git", e) == false && has_folder?(".hg", e) == false }
+            begin
+                fpath = File.realpath(fpath).force_encoding("binary")
+                fsize = File.size(fpath)
+
+                if fsize == 0
+                    nskipfiles += 1
+                    puts "[SKIP]: #{fpath}"
+                    next
+                end
+
+                # Lazy index the file.
+                if h[fsize].nil?
+                    h[fsize] = fpath
+                    needupdate = true
+                    nnewfiles += 1
+                    puts "[NEW]: #{fpath} nil"
+                else
+                    if h[fsize].is_a?(String)
+                        p = h[fsize]
+                        if p == fpath
+                            ndupfiles += 1
+                            puts "[DUP]: #{fpath}"
+                            puts "       #{p}"
+                            next
+                        end
+
+                        d = Digest::MD5.hexdigest(File.read(p))
+                        h[fsize] = { d => FileMeta.new(fsize, p, d) }
+                        needupdate = true
+                    end
+                    digest = Digest::MD5.hexdigest(File.read(fpath))
+                    if h[fsize][digest].nil?
+                        h[fsize][digest] = FileMeta.new(fsize, fpath, digest)
+                        needupdate = true
+                        nnewfiles += 1
+                        puts "[NEW]: #{fpath} #{digest}"
+                    else
+                        ndupfiles += 1
+                        puts "[DUP]: #{fpath}"
+                        puts "       #{h[fsize][digest].path}"
+                    end
+                end
+            rescue StandardError => e
+                nfailfiles += 1
+                puts "[FAIL]: file=\"#{fpath}\" #{e.message}"
+            end
+        end
+        t1 = Time.now
+        puts "Indexing completed in #{(t1-t0).round(2)}s."
+
+        puts "Indexed:"
+        puts "     NEW: #{nnewfiles}"
+        puts "     DUP: #{ndupfiles}"
+        puts "    SKIP: #{nskipfiles}"
+        puts "    FAIL: #{nfailfiles}"
+
+        if needupdate && options[:updatedb]
+            puts "Updating DB #{options[:dbfile]} ..."
+            t0 = Time.now
+            File.write(options[:dbfile], YAML::dump(h))
+            t1 = Time.now
+            puts "Updated DB in #{(t1-t0).round(2)}s."
+        else
+            puts "Skip updating DB."
+        end
+
+    end
+
+
+    module_function :cmd_index
+
+end

data/lib/fmanager/fmatch.rb

@@ -0,0 +1,11 @@
+module FManager
+
+    def has_folder?(folder, path)
+        pos = (path =~ /^#{folder}\/|^#{folder}$|\/#{folder}\/|\/#{folder}$/)
+        pos != nil
+    end
+
+
+    module_function :has_folder?
+
+end

data/lib/fmanager/fmeta.rb

@@ -0,0 +1,15 @@
+module FManager
+
+    class FileMeta
+        attr_reader :fsize
+        attr_reader :digest
+        attr_reader :path
+
+        def initialize(fsize, path, digest)
+            @fsize = fsize
+            @digest = digest
+            @path = path
+        end
+    end
+
+end

data/lib/fmanager/version.rb

@@ -1,3 +1,3 @@
 module FManager
-    VERSION = '0.0.0'
+    VERSION = '0.0.1'
 end

data/test/test_cmd_index.rb

@@ -0,0 +1,147 @@
+require 'tempfile'
+require 'test/unit'
+require 'yaml'
+
+require 'fmanager'
+
+
+RB = "ruby -Ilib"
+FM = "bin/fm"
+DATA = "test/data"
+NEWDATA = "test/newdata"
+
+
+class TestCmdIndex < Test::Unit::TestCase
+
+    include FManager
+
+
+    def setup
+        @stdout = Tempfile.new("test.out")
+        @stderr = Tempfile.new("test.err")
+        @stdout.close
+        @stderr.close
+    end
+
+    def teardown
+        @stdout.unlink
+        @stderr.unlink
+    end
+
+
+    def test_no_arguments
+        assert( execute_fm_index() )
+        assert_match(/usage: fm index/, File.read(@stderr.path))
+    end
+
+
+    def test_help_option
+        assert( execute_fm_index("-h") )
+        assert_match(/Show this message./, File.read(@stdout.path))
+    end
+
+
+    def test_default_db
+        fname = "fmetadata.db"
+        assert( File.exists?(fname) == false, "File #{fname} already exists. Test is unable to continue." )
+        assert( execute_fm_index("#{DATA}") )
+        assert_test_db_content(fname)
+        File.unlink(fname)
+    end
+
+
+    def test_db_file_option
+        dbfile = Tempfile.new("test.db")
+        dbfile.close
+        fname = dbfile.path
+        assert( execute_fm_index("-d #{fname} #{DATA}") )
+        assert_test_db_content(fname)
+        dbfile.unlink
+    end
+
+
+    def test_no_update_db_option
+        data = "hello world"
+        dbfile = Tempfile.new("test.db")
+        dbfile.write(data)
+        dbfile.close
+        fname = dbfile.path
+        assert( execute_fm_index("-n -d #{fname} #{DATA}") )
+        assert_match( /Indexing completed/, File.read(@stdout.path) )
+        fdata = File.read(fname)
+        assert_equal(data, fdata)
+        dbfile.unlink
+    end
+
+
+    def test_update_db_option
+        dbfile = Tempfile.new("test.db")
+        dbfile.close
+        fname = dbfile.path
+        assert( execute_fm_index("-u -d #{fname} #{DATA}") )
+        assert_test_db_content(fname)
+        dbfile.unlink
+    end
+
+
+    def test_existing_db
+        dbfile = Tempfile.new("test.db")
+        dbfile.close
+        fname = dbfile.path
+        assert( execute_fm_index("-d #{fname} #{DATA}") )
+        assert( execute_fm_index("-d #{fname} #{NEWDATA}") )
+        h = YAML::load( File.read(fname) )
+
+        # file0.txt
+        fsize = 13
+        digest = "8b362a13f2467448f69c826a7450cf3b"
+        assert(f0 = h[fsize][digest])
+        assert_equal(fsize, f0.fsize)
+        assert_equal(digest, f0.digest)
+        assert_match(/newdata\/file0.txt/, f0.path)
+        dbfile.unlink
+
+        assert_equal(4, h.size)
+    end
+
+
+    def execute_fm_index(argv = "")
+        system("#{RB} #{FM} index #{argv} > #{@stdout.path} 2> #{@stderr.path}")
+    end
+
+    def assert_test_db_content(path)
+        assert( File.exists?(path) )
+        h = YAML::load( File.read(path) )
+
+        # data0.txt and data2.txt
+        fsize = 53
+        digest = "be4af635154944fdf305dcb91a9613ba"
+        assert(f0 = h[fsize][digest])
+        assert_equal(fsize, f0.fsize)
+        assert_equal(digest, f0.digest)
+        assert_match(/data\/data0.txt|data\/data2.txt/, f0.path)
+
+        # data1.txt
+        fsize = 13
+        digest = "2632561a9eb94e0d05e3e032cc5adeed"
+        assert(f1 = h[fsize][digest])
+        assert_equal(fsize, f1.fsize)
+        assert_equal(digest, f1.digest)
+        assert_match(/data\/data1.txt/, f1.path)
+
+        #data3.txt
+        fsize = 13
+        digest = "412cb2c7b6a2cd60c55e3548066cbdeb"
+        assert(f3 = h[fsize][digest])
+        assert_equal(fsize, f3.fsize)
+        assert_equal(digest, f3.digest)
+        assert_match(/data\/data3.txt/, f3.path)
+
+        # data4.txt
+        fsize = 77
+        assert_match(/data\/data4.txt/, h[fsize])
+
+        assert_equal(3, h.size)
+    end
+
+end

data/test/test_fmatch_hasfolder.rb

@@ -0,0 +1,46 @@
+require 'test/unit'
+
+require 'fmanager'
+
+
+class TestFMatchHasFolder < Test::Unit::TestCase
+
+    include FManager
+
+
+    def test_absolute_path
+        path = "/abc/defg/hij"
+
+        # Full folder name.
+        assert( has_folder?("abc", path) )
+        assert( has_folder?("defg", path) )
+        assert( has_folder?("hij", path) )
+
+        # Partial folder name.
+        assert( has_folder?("efg", path) == false )
+        assert( has_folder?("ac", path) == false )
+
+        # Non-existing name.
+        assert( has_folder?("jih", path) == false )
+        assert( has_folder?("xyz", path) == false )
+    end
+
+    def test_relative_path_trailing_slash
+        path = "abc0123/defg/hij456/"
+
+        # Full folder name.
+        assert( has_folder?("abc0123", path) )
+        assert( has_folder?("defg", path) )
+        assert( has_folder?("hij456", path) )
+
+        # Partial folder name.
+        assert( has_folder?("abc012", path) == false )
+        assert( has_folder?("efg", path) == false )
+        assert( has_folder?("456", path) == false )
+
+        # Non-existing name.
+        assert( has_folder?("xyz", path) == false )
+        assert( has_folder?("aabbc", path) == false )
+    end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fmanager
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.1
 platform: ruby
 authors:
 - ShinYee
@@ -21,11 +21,16 @@ extra_rdoc_files: []
 files:
 - lib/fmanager.rb
 - lib/fmanager/version.rb
+- lib/fmanager/fmatch.rb
+- lib/fmanager/fmeta.rb
+- lib/fmanager/findex.rb
 - COPYING
 - fmanager.gemspec
 - Rakefile
 - history.txt
 - readme.md
+- test/test_fmatch_hasfolder.rb
+- test/test_cmd_index.rb
 - test/test_version.rb
 - bin/fm
 homepage: http://github.com/xman/ruby-fmanager
@@ -53,4 +58,6 @@ signing_key:
 specification_version: 4
 summary: Managing files over multiple storage resources.
 test_files:
+- test/test_fmatch_hasfolder.rb
+- test/test_cmd_index.rb
 - test/test_version.rb