bc3 0.1.0 → 0.1.1

data/lib/bc3/snapshot.rb

@@ -1,3 +1,20 @@
+#~ %{
+                #~ file
+                #~ folder/subfolder2/
+                #~ folder/file1
+                #~ folder/subfolder/subfile1
+#~ }.each_line{|line|
+#~ p line.strip
+#~ x = line.strip.split(%r{(.+?[\\\/])})
+#~ x.delete('')
+#~ p x
+#~ p '========'
+#~ }
+
+#~ __END__
+
+$:.unshift('..') if $0 == __FILE__  #only for test
+  
 require 'bc3/helper'
 module BC3
 =begin rdoc
@@ -14,6 +31,7 @@ Container for a snapshot.
     
     $log.debug("Create base folder for snapshot #{path}")
     @basefolder = Folder.new('SnapshotRoot', @timestamp)
+    @index = nil
   end
 =begin rdoc
 Create a snapshot from a hash.
@@ -23,7 +41,7 @@ A snapsot-hash must contain:
 * content - array of folders (see Folder.newh) and files (File.new)
 * timestamp (optional)
 =end
-  def self.newh( data )
+  def self.new_hash( data )
     $log.info("Build Snapshot from hash")
     raise ArgumentError, "No hash given" unless data.is_a?(Hash)
     raise ArgumentError, "snapshot name missing" unless data.has_key?(:snapshot)
@@ -44,9 +62,8 @@ A snapsot-hash must contain:
   end #newh
 =begin rdoc
 Create a snapshot from a directory.
-
 =end
-  def self.newd( dirname )
+  def self.new_filesystem( dirname )
     $log.info("Build Snapshot from directory #{dirname}")
     
     #~ raise ArgumentError, "No hash given" unless data.is_a?(Hash)
@@ -64,9 +81,49 @@ Create a snapshot from a directory.
     }
     snapshot
   end #newh
+=begin rdoc
+Simple interface to create a snapshot.
+
+Snapshot structure is a list with a filename (including path) on a line.
+
+Empty folders end with a slash.
+
+  file
+  folder/file1
+  folder/subfolder/subfile1
+  folder/subfolder2/
+
+=end
+  def self.new_filelist( startpath, filelist )
+    snap = self.new( startpath )
+    filelist.strip.each_line{|line|
+      line.strip!
+      basename = ::File.basename(line)
+      #Decide, if we add a (empty) folder or a file.
+      entry = ( line[-1,1] == '/' ? 
+                        Folder.new(basename) 
+                      : File.new(filename: basename, filesize: 0)
+                  )
+      path = ::File.dirname(line)
+      if path == '.'
+        snap << entry
+      else
+        snap.basefolder.add_with_path(path, entry)
+      end
+    }
+    snap
+  end #self.new_filelist
+
+  #Keep old definition
+  class << self
+    alias :newd :new_filesystem
+    alias :newh :new_hash
+  end
 
   #homepath of the snapshot
   attr_reader :path
+  #Content of the snapshot
+  attr_reader :basefolder
   #Time stamp from snapshot. Default 'now'
   attr_reader :timestamp
   
@@ -74,32 +131,127 @@ Create a snapshot from a directory.
 Add content (folders/files) to snapshot.
 =end
   def << (content)
+      reset_index
       @basefolder << content
   end
 =begin rdoc
+Get content of the folder.
+
+If you request a folder, the path must end with a '/'
+=end
+    def [](path)
+      build_index() unless @index
+      @index[path]
+    end
+=begin rdoc
 Loop on content of the folder.
 
-Options see BC3::Folder#each
+Options:
+* :files (default)
+* :folders
+* :rootonly - only root, no subdirectories
+See also BC3::Folder#each (in combination with BC3::Snapshot#basefolder).
 =end
     def each(*options)
+      build_index() unless @index
+      options = [:files] if options.empty?
+      if options.include?(:flat)
+        $log.warn("each-option :flat should be rootonly")
+        options << :rootonly
+      end
+
+      result = {}
+      @index.each{|key,content|
+        next if options.include?(:rootonly) and key =~ %r{/.} #no subdirectory
+        case content
+          when File
+            result[key] = content if options.include?(:files)
+          when Folder
+            result[key] = content if options.include?(:folders)
+          else
+            raise "Internal error"
+          end
+      }
+      
       if block_given?
-        @basefolder.each(*options){|key, content| yield key, content }
+        result.each{|key, content| yield key, content }
       else
-        @basefolder.each
+        result
       end
+    end #each
+=begin rdoc
+Reset the index.
+The index will be regenerated when needed
+(See BC3::Snapshot#build_index)
+
+Called from folders of the snapshot, when they get new content.
+=end
+  def reset_index(rebuild = false)
+    $log.debug("Delete index")
+    @index = nil
+    build_index() if rebuild
+  end
+=begin rdoc
+Build an index.
+
+The index allows a faster process on each.
+
+When the content changes (or content of one of the folders
+in the snapshot) then the index is deleted.
+See BC3::Snapshot#reset_index
+
+
+=end
+    def build_index()
+      $log.debug("Build index")
+      @index = {}
+      @basefolder.each(:recursive,:folders, :files){|key, content| 
+        @index[key] = content
+        content.snapshotpath[self] = key
+      }
+      @index
     end
 
 =begin rdoc
+Save the snapshot.
+
+Depending on extension you get different results:
+* *.bcss Beyond compare snapshot format
+* *.yaml Yaml file.
+
+You may save the snapshot as a yaml file, modify the data and 
+create a new bcss-file.
+
+  bc3 = BC3::Snapshot.new()
+  ...
+  bc3.save('my_snapshot.yaml')
+  
+  ... modify the yaml file
+  
+  bc3 = BC3::Snapshot.new_hash(YAML.load(File.read('my_snapshot.yaml'))
+  bc3.save('my_snapshot.bcss')
+
 =end
     def save( filename, compressed = nil ) 
       $log.debug("Prepare snapshot for #{filename}")
-      #Check if compressed or uncompressed output wanted
-      compressed = ( filename =~ /\.bcssx/ ) if compressed.nil?
-      #Must be binary, else a \n get's \r\n under windows.
-      ::File.open(filename,'wb'){|f|
-        f << bcss( compressed )
-      }
-      $log.info("Saved snapshot as #{filename}")
+      
+      case filename
+        when /\.ya?ml/
+          ::File.open(filename,'w'){|f|
+            f << to_hash.to_yaml
+          }
+          $log.info("Saved snapshot as #{filename}")
+        when /\.bcss/
+          #Check if compressed or uncompressed output wanted
+          compressed = ( filename =~ /\.bcssx/ ) if compressed.nil?
+          #Must be binary, else a \n get's \r\n under windows.
+          ::File.open(filename,'wb'){|f|
+            f << bcss( compressed )
+          }
+          $log.info("Saved snapshot as #{filename}")
+        else
+          raise ArgumentError, "Undefined filetype #{::File.extname(filename)}"
+        end
     end
 =begin rdoc
 Collect the data in a hash.
@@ -260,5 +412,81 @@ This part may be packed.
       }
       data
     end
+=begin rdoc
+Create a little statistic.
+
+* Count of files and folders
+* Collection of duplicates, grouped by crc
+=end
+    def statistic()
+      count = Hash.new(0)
+      crc   = Hash.new()
+      #feature idea: duplicates_by_name . Better: duplicates_by_filename , duplicates_by_dirname ?
+      
+            
+      @basefolder.each(:recursive,:folders, :files){|key, element|
+        case element
+          when Folder
+            count['folders'] += 1
+          when File
+            count['files'] += 1
+            (crc[element.crc] ||= [] ) << key unless element.crc.nil? 
+          else
+            raise "Internal error #{element}"
+          end
+      }
+      
+      count['duplicates_by_crc'] = {} 
+      crc.each{|key, values|
+        count['duplicates_by_crc'][key] = values if values.size > 1
+      }
+      count.delete('duplicates_by_crc') if count['duplicates_by_crc'].empty?
+      count
+    end
   end #Snapshot
 end #module BC3
+
+
+#testcode for immediate test
+if $0 == __FILE__
+  $:.unshift('..')
+  require 'bc3'
+  require 'yaml'
+    bc3 = BC3::Snapshot.new_filelist( 'dir', %{
+                file
+                folder/subfolder2/
+                folder/file1
+                folder/subfolder/subfile1
+      }
+    )
+    puts bc3.each(:files, :folders ).keys
+    exit
+    #~ bc3 = BC3::Snapshot.new_filesystem('.')
+    bc3 << folder = BC3::Folder.new('test')
+    
+    bc3 = BC3::Snapshot.new_filelist( 'dir', %{
+                file
+                folder/file1
+                folder/subfolder/subfile1
+                folder/subfolder2/
+      } )
+    #extend object
+    class << bc3;      attr_reader :index;    end    
+    bc3.build_index
+
+
+    puts "hash: #{bc3.index.inspect}"
+    #Check changes in root
+    folder =  ( bc3 << BC3::Folder.new('folder') )
+    puts "nil: #{bc3.index.inspect}"
+    puts "fnil: #{folder.snapshotpath}"
+    
+    bc3.build_index
+    puts "hash: #{bc3.index.inspect}"
+    puts "fhash: #{folder.snapshotpath}"
+
+    #check changes in subdirectories
+    folder << BC3::Folder.new('folder')
+    puts "nil: #{bc3.index.inspect}"
+
+end

data/unittest/unittest_bc3_file.rb

@@ -24,6 +24,12 @@ class Test_file < Test::Unit::TestCase
     assert_raise(ArgumentError){ BC3::File.new( 'f' )}
     assert_raise(ArgumentError){ BC3::File.new( 1 )}
   end
+  def test_basename()
+    file = BC3::File.new( filename: 'x', filesize: 5)
+    assert_equal('x', file.filename )
+    assert_equal('x', file.basename )
+    assert_equal(file.filename, file.basename )
+  end  
   def test_hash()
     assert_equal(   { filename: 'f', filesize: 5, timestamp: $timestamp, attributes: BC3::Attrib::Archive, crc: nil},
         BC3::File.new( filename: 'f', filesize: 5, timestamp: $timestamp ).to_hash

data/unittest/unittest_bc3_folder.rb

@@ -10,11 +10,19 @@ $log.level = Log4r::FATAL
 $timestamp= Time.local(2010,1,1)
 
 class Test_folder < Test::Unit::TestCase
+  def test_basename()
+    folder = BC3::Folder.new('x')
+    assert_equal('x', folder.dirname )
+    assert_equal('x', folder.basename )
+    assert_equal(folder.dirname, folder.basename )
+  end
   def test_add_content()
     folder = BC3::Folder.new('x')
     assert_nothing_raised{ folder << BC3::Folder.new('dir1')}
     assert_nothing_raised{ folder << BC3::File.new( filename: 'file1', filesize: 1) }
+    
     assert_raise(ArgumentError){ folder << 1 }
+    assert_raise(ArgumentError){ folder << BC3::Snapshot.new('x')}
     
     assert_equal(2, folder.each.size)
   end
@@ -134,18 +142,18 @@ class Test_folder < Test::Unit::TestCase
     assert_equal(2, folder.each(:files).size)
     assert_equal(1, folder.each(:folders).size)
     
-    assert_equal(%w{subdir file1 file2}, folder.each.keys)
+    assert_equal(%w{subdir/ file1 file2}, folder.each.keys)
     assert_equal(%w{file1 file2}, folder.each(:files).keys)
-    assert_equal(%w{subdir}, folder.each(:folders).keys)
+    assert_equal(%w{subdir/}, folder.each(:folders).keys)
     
     assert_equal(1, folder.each(:folders, :recursive).size)
     assert_equal(3, folder.each(:files, :recursive).size)
     assert_equal(4, folder.each(:folders,:files, :recursive).size)
     
     
-    assert_equal(%w{subdir}, folder.each(:folders, :recursive).keys)
+    assert_equal(%w{subdir/}, folder.each(:folders, :recursive).keys)
     assert_equal(%w{subdir/subfile file1 file2}, folder.each(:files, :recursive).keys)
-    assert_equal(%w{subdir subdir/subfile file1 file2}, folder.each(:folders,:files, :recursive).keys)
+    assert_equal(%w{subdir/ subdir/subfile file1 file2}, folder.each(:folders,:files, :recursive).keys)
     
   end
 
@@ -170,10 +178,28 @@ class Test_folder < Test::Unit::TestCase
     assert_equal(3, folder.each(:files, :recursive).size)
     assert_equal(5, folder.each(:folders,:files, :recursive).size)
     
-    assert_equal(%w{subdir subdir/subsubdir}, folder.each(:folders, :recursive).keys)
+    assert_equal(%w{subdir/ subdir/subsubdir/}, folder.each(:folders, :recursive).keys)
     assert_equal(%w{subdir/subsubdir/subsubfile subdir/subfile file}, folder.each(:files, :recursive).keys)
-    assert_equal(%w{subdir subdir/subsubdir subdir/subsubdir/subsubfile subdir/subfile file}, folder.each(:folders,:files, :recursive).keys)
+    assert_equal(%w{subdir/ subdir/subsubdir/ subdir/subsubdir/subsubfile subdir/subfile file}, folder.each(:folders,:files, :recursive).keys)
+    
+  end
+  
+  def test_add_with_path()
+    folder = BC3::Folder.new('base')
+    assert_equal( %w{},folder.each(:recursive,:files).keys )
+    
+    folder.add_with_path('dir2', BC3::File.new(filename: 'test.txt', filesize: 5 ))
+    assert_equal( %w{dir2/test.txt},folder.each(:recursive,:files).keys )
+    
+    folder.add_with_path('dir2/subdir', BC3::File.new(filename: 'test.txt', filesize: 5 ))
+    assert_equal( %w{dir2/test.txt dir2/subdir/test.txt},folder.each(:recursive,:files).keys )
+    
+    folder.add_with_path('dir4/subdir', BC3::File.new(filename: 'test.txt', filesize: 5 ))
+    assert_equal( %w{dir2/test.txt dir2/subdir/test.txt dir4/subdir/test.txt},folder.each(:recursive,:files).keys )
     
+    #Added at end - sorting not necessary.
+    folder.add_with_path('dir0', BC3::File.new(filename: 'test.txt', filesize: 5 ))
+    assert_equal( %w{dir2/test.txt dir2/subdir/test.txt dir4/subdir/test.txt dir0/test.txt},folder.each(:recursive,:files).keys )
   end
 end
 

data/unittest/unittest_bc3_merge.rb

@@ -46,13 +46,16 @@ class Test_merge < Test::Unit::TestCase
           }
         ],
       }
-  TestFiles = %w{snap1.bcss snap2.bcss
+  TestFiles = %w{
+          snap1.bcss snap2.bcss
+          snap1.yaml
           merge_i.bcss merge_b_1.bcss merge_b_2.bcss merge_r.bcss 
     }
       
   def self.startup
     TestFiles.each{|f| File.delete(f) if File.exist?(f)}
     BC3::Snapshot.newh(Snap1).save('snap1.bcss')
+    #~ BC3::Snapshot.newh(Snap1).save('snap1.yaml')
     BC3::Snapshot.newh(Snap2).save('snap2.bcss')
   end
   

data/unittest/unittest_bc3_search.rb

@@ -0,0 +1,231 @@
+gem 'test-unit'
+require 'test/unit'
+
+$:.unshift('../lib')
+require 'bc3'
+$log.level = Log4r::WARN
+
+#Start with local sources
+$call_bc3_search = 'ruby -I ../lib ../bin/bc3_search.rb'
+class String
+  #clean String from time dependend output.
+  def clean
+    self.strip.gsub(/\d{4,4}-\d{2,2}-\d{2,2} \d{2,2}:\d{2,2}:\d{2,2}\.\d+ \+\d\d:\d\d/, '2011-01-01 00:00:00 +01:00')
+    #~ :timestamp: 2011-01-28 21:37:05.265625 +01:00
+  end
+end
+#~ puts ":timestamp: 2011-01-28 21:37:05.265625 +01:00".clean
+
+
+class Test_search < Test::Unit::TestCase
+  Testdata =  YAML.load(%{
+      :snapshot: c:/my_dir
+      :content: 
+      - :dirname: dir
+        :timestamp: 2011-01-01 12:00:00 +01:00
+        :content: 
+        - :filename: file_15.txt
+          :filesize: 1
+          :crc: 15
+          :timestamp: 2011-01-01 12:00:00 +01:00
+        - :filename: file_with_crc_16.txt
+          :filesize: 1
+          :crc: 16
+          :timestamp: 2011-01-01 12:00:00 +01:00
+      - :filename: file_with_crc_15.txt
+        :filesize: 1
+        :crc: 15
+        :timestamp: 2011-01-01 12:00:00 +01:00
+    })
+
+  TestFiles = %w{test_search.bcss test_search_2.bcss test_search_extract.bcss}
+      
+  def self.startup
+    TestFiles.each{|f| File.delete(f) if File.exist?(f)}
+    Dir['*.bcss'].each{|bcssfile|
+      raise "Snapshot exists before test run"
+    }
+    BC3::Snapshot.new_hash(Testdata).save('test_search.bcss')
+    Testdata[:content].first[:content].pop #delete file_with_crc_16.txt
+    BC3::Snapshot.new_hash(Testdata).save('test_search_2.bcss')
+  end
+  
+  def self.shutdown
+    TestFiles.each{|f| File.delete(f) if File.exist?(f)}
+  end
+
+  def test_search_no_bcss()
+    assert_equal(%{
+Start bc3_search for pattern <aa>
+No Snapshot found
+}.strip,  `#{$call_bc3_search} 'aa' `.strip)
+
+    assert_equal(%{
+Start bc3_search for pattern <aa>
+Snapshot x.bcss not found
+}.strip,  `#{$call_bc3_search} 'aa' x.bcss`.strip)
+
+  end
+  #
+  #Check pattern to regexp conversion
+  def test_pattern()
+    assert_equal(%{
+Start bc3_search for pattern <xx.x\\.txt>
+Analyse test_search.bcss (c:/my_dir)
+}.strip, `#{$call_bc3_search} 'xx?x.txt' test_search.bcss`.strip)
+
+    assert_equal(%{
+Start bc3_search for pattern <xx.*\\.txt>
+Analyse test_search.bcss (c:/my_dir)
+}.strip, `#{$call_bc3_search} 'xx*.txt' test_search.bcss`.strip)
+  end
+  #No regexp-conversion with parameter -r
+  def test_regexp()
+    assert_equal(%{
+Start bc3_search for pattern <xx*.txt>
+Analyse test_search.bcss (c:/my_dir)
+}.strip, `#{$call_bc3_search} -r 'xx*.txt' test_search.bcss`.strip)
+
+    assert_equal(%{
+Start bc3_search for pattern <crc_[1]\\d\\.txt>
+Analyse test_search.bcss (c:/my_dir)
+	dir/file_with_crc_16.txt
+	file_with_crc_15.txt
+Found   2 files in test_search.bcss
+
+}.strip, `#{$call_bc3_search} -r 'crc_[1]\\d\\.txt' test_search.bcss`.strip)
+
+  end
+
+  def test_search_dir()
+    assert_equal(%{
+Start bc3_search for pattern <dir>
+Analyse test_search.bcss (c:/my_dir)
+	dir/
+Found   1 folder in test_search.bcss
+}.strip, `#{$call_bc3_search} 'dir' test_search.bcss`.strip)
+  end
+  def test_search_file()
+    assert_equal(%{
+Start bc3_search for pattern <file_15>
+Analyse test_search.bcss (c:/my_dir)
+	dir/file_15.txt
+Found   1 file in test_search.bcss
+
+}.strip, `#{$call_bc3_search} 'file_15' test_search.bcss`.strip)
+    assert_equal(%{
+Start bc3_search for pattern <file.*15>
+Analyse test_search.bcss (c:/my_dir)
+	dir/file_15.txt
+	file_with_crc_15.txt
+Found   2 files in test_search.bcss
+
+}.strip, `#{$call_bc3_search} 'file*15' test_search.bcss`.strip)
+  end
+  def test_yaml()
+    assert_equal(%{Start bc3_search for pattern <.*>
+Analyse test_search.bcss (c:/my_dir)
+--- 
+:snapshot: c:/my_dir_extract
+:timestamp: 2011-01-01 00:00:00 +01:00
+:content: 
+- :dirname: dir
+  :timestamp: 2011-01-01 12:00:00 +01:00
+  :attributes: 16
+  :content: 
+  - :filename: file_15.txt
+    :filesize: 1
+    :crc: 15
+    :attributes: 32
+    :timestamp: 2011-01-01 12:00:00 +01:00
+  - :filename: file_with_crc_16.txt
+    :filesize: 1
+    :crc: 16
+    :attributes: 32
+    :timestamp: 2011-01-01 12:00:00 +01:00
+- :filename: file_with_crc_15.txt
+  :filesize: 1
+  :crc: 15
+  :attributes: 32
+  :timestamp: 2011-01-01 12:00:00 +01:00}, 
+  `#{$call_bc3_search} -y '*' test_search.bcss`.clean)
+  end
+  def test_duplicates()
+    #Duplicate search, pattern must be '*'
+    assert_equal(%{
+Start bc3_search for pattern <.*>
+Analyse test_search.bcss (c:/my_dir)
+	dir/file_15.txt
+	file_with_crc_15.txt
+Found   2 files in test_search.bcss
+
+}.strip, `#{$call_bc3_search} -d '*' test_search.bcss`.strip)
+
+    #Duplicate search in combination with pattern
+    assert_equal(%{
+Start bc3_search for pattern <xx.*\\.txt>
+Analyse test_search.bcss (c:/my_dir)
+
+}.strip, `#{$call_bc3_search} -d 'xx*.txt' test_search.bcss`.strip)
+  end
+  
+  def test_duplicates_global()
+    assert_equal(%{
+Start bc3_search for pattern <.*>
+Restrict on global duplicates
+Analyse test_search.bcss (c:/my_dir)
+	dir/file_15.txt
+	file_with_crc_15.txt
+Found   2 files in test_search.bcss
+
+Analyse test_search_2.bcss (c:/my_dir)
+	dir/file_15.txt
+	file_with_crc_15.txt
+Found   2 files in test_search_2.bcss
+
+}.strip, `#{$call_bc3_search} '*' test_search*.bcss -D`.strip)
+  end
+
+  def test_result_as_snapshot()
+    assert_false(File.exist?('test_search_extract.bcss'))
+    assert_equal(%{
+Start bc3_search for pattern <xx.*>
+Analyse test_search.bcss (c:/my_dir)
+Save result in test_search_extract.bcss
+
+}.strip, `#{$call_bc3_search} -b extract 'xx*' test_search.bcss`.strip)
+    assert_true(File.exist?('test_search_extract.bcss'))
+    
+    call_result = `#{$call_bc3_search} 'xx*' test_search.bcss -b`
+    regex = %r{Start bc3_search for pattern <xx\.\*>
+Analyse test_search.bcss \(c:/my_dir\)
+Save result in (test_search_extract_\d\d\d\d-\d\d-\d\d_\d\d\d\d.bcss)
+}m
+    assert_match(regex, call_result)
+    regex =~ call_result
+    assert_true(File.exist?($1))
+    
+    #delete snapshots, else other test will get problems
+    File.delete($1)
+    File.delete('test_search_extract.bcss')
+  end
+end
+
+def mk_test(name, options)
+    output = `#{$call_bc3_search} #{options}`
+    puts <<xx
+====Neuer Test #{name} ====
+!#{options}
+#{output}
+
+  def test_#{name}()
+    assert_equal(%{
+#{output}}.strip, `\#{$call_bc3_search} #{options}`.strip)
+  end
+====Neuer Test #{name} ====
+xx
+end
+#~ Test_search.startup
+#~ mk_test('bcss_result', "-b result.bcss 'xx*.txt' test_search.bcss") if false
+#~ mk_test('duplicates_global', "'*' test_search*.bcss -D")
+#~ exit  #no tests