bc3 0.1.1 → 0.1.2

data/bin/bc3_search.rb

@@ -104,12 +104,6 @@ return the result in a yaml-list.
 
 =end
 
-=begin
-fixme/test:
-duplicates
-save as snapshot.
-=end
-
 $testmode = false
 #~ $testmode = true
 if $testmode
@@ -123,12 +117,12 @@ puts "!!Testmode"
 $:.unshift('../lib')
 end
 
+#~ gem 'bc3', '= 0.1.2beta'
 require 'optparse'
 require 'bc3'
 #~ $log.level = Log4r::DEBUG
 $log.level = Log4r::WARN
 
-
 #Collector
 $options = {
   duplicates: false,

data/changes.rd

@@ -0,0 +1,37 @@
+=Change History
+==0.1.0: 2011-01-24 Initial version
+==0.1.1: 2011-01-30
+* New: BC3::Snapshot.new_filelist
+* New: BC3::Folder#add_with_path
+* New: bin/bc_search.rb
+* Changed BC3::Snapshot.newh -> BC3::Snapshot.new_hash (newh as alias)
+* Changed BC3::Snapshot.newd -> BC3::Snapshot.new_filesystem (newd as alias)
+* BC3::Snapshot.save saves yaml or bcss file
+* bin/bc_merge.rb can handle yaml-files
+==0.1.2 2011-02-20
+* BC3::SnapshotParse: Support UTF for linux files
+* BC3::Snapshot#utf: Added flag. Allows creation of Linux-UTF-snapshots.
+* BC3::Snapshot: Support UTF for linux files (Snapshot uses UTF-8).
+* BC3::Snapshot.uncompress: Added
+
+
+
+=Known problems/Todos
+
+==Missing code
+* Snapshot.to_yaml: Add UTF-texts, utf-flag
+* Snapshots/Win32 with Unicode Creation error
+
+==Misc
+* No compression of snapshots (uncompression works)
+* test_reconstruct.rb make little timestamp changes (accuracy of Time?)
+
+Tests require BC3-folder comparison in background.
+* Missing unittest for bcss-generation
+* Missing unittest for compressed bcss.
+* Missing unittest for uncompressed bcss.
+
+==Not covered
+The following features of beyond compare are not covered by this tool:
+
+* Symbolic links

data/examples/results/readme

@@ -0,0 +1 @@
+This folder ist the target for the examples.

data/examples/scootersoftware/readme

@@ -0,0 +1 @@
+All examples in this directory are copies from:

data/examples/test_reconstruct.rb

@@ -0,0 +1,49 @@
+=begin
+Take the original examples from becond compare
+(http://www.scootersoftware.com/vbulletin/showthread.php?t=7210),
+parse them and reconstruct them again.
+
+=end
+$:.unshift('../lib')
+
+require 'bc3'
+require 'yaml'
+
+def read_and_save(filename, *options )
+  puts "Start #{options.inspect} for #{filename}"
+  options.each{|opt|
+    puts "undefined option #{opt}" unless %w{bcss xxx yaml yaml_stdout uncompressed}.include?(opt.to_s)
+  }
+  
+  target = File.join("results", File.basename(filename))
+
+  snap = BC3::SnapshotParser.new(filename)
+  
+  puts snap.snapshot.to_hash.to_yaml  if options.include?(:yaml_stdout)
+  snap.snapshot.save(target.sub('.bcss', '.yaml')) if options.include?(:yaml)
+  
+  snap.snapshot.save(target) if options.include?(:bcss)
+  
+  if options.include?(:uncompressed)
+    BC3::Snapshot.uncompress(filename, target.sub('.bcss', '_uncompressed.bcss'))
+  end
+
+  #Copy result to enable binary compare
+  FileUtils.copy(filename, target.sub('\.bcss', '.xxx') ) if options.include?(:xxx)
+
+end
+#ok with minor binary differences (time stamp in milliseconds-steps)
+read_and_save('scootersoftware/UncompressedSample.bcss', :bcss)
+
+#~ $log.level=Log4r::DEBUG
+$log.level=Log4r::WARN
+$log.level=Log4r::INFO
+#~ read_and_save('scootersoftware/UnicodeFilenames(Linux).bcss', :bcss)
+#~ read_and_save('scootersoftware/UnicodePath(Linux).bcss', :bcss)
+
+=begin
+2010-02-01: Will not work correct
+=end
+#--
+#~ read_and_save('scootersoftware/UnicodeFilenames(Win32).bcss', :uncompressed, :bcss, :yaml)
+#~ read_and_save('scootersoftware/UnicodePath(Win32).bcss', :bcss)

data/lib/bc3.rb

@@ -85,5 +85,15 @@ require 'bc3/time'
 require 'bc3/parse'
 
 module BC3
-  VERSION = '0.1.1'
-end
+  VERSION = '0.1.2'
+end
+
+__END__
+fixme logger/sublogger
+
+Snapshot definiert logger, vererbt an subloggers dir/file
+gem 'log4r', '>= 1.1.9'
+require 'log4r' 
+log = Logger.new('log')
+log2 = Logger.new('log::2::b')
+

data/lib/bc3/file.rb

@@ -47,12 +47,13 @@ Must contain KEYS and supports KEYS_OPTIONAL
     end
     
 =begin rdoc
-Create a folder from file system.
+Creates a BC3::File-object, based on data from file system.
 
 The argument must contain:
-* dirname
+* filename
+* flag, if crc schould be added (default: true)
 =end
-    def self.new_by_filename( filename )
+    def self.new_by_filename( filename, with_crc = true )
       $log.debug("Build file #{filename} from file system")
       
       #fixme: get attributes from file system
@@ -62,14 +63,15 @@ The argument must contain:
       settings = {
         filename: filename,
         filesize: ::File.size(filename),
-        #~ attributes: ::File.stat(filename),
+        attributes: Attrib.fileattrib(filename),
         #The timestamp must be 'local time' -> ignore local time zone for generation
         timestamp: ::File.mtime(filename) + Time.now.gmt_offset,
       }
       
-      ::File.open( filename, 'r'){|f| 
+      #Binary read needed, else the crc may differ from BC3
+      ::File.open( filename, 'rb'){|f| 
         settings[:crc]  = Helper::crc32(f.read)
-      } #fixme crc if ...
+      } if with_crc
       return File.new( settings )
     end
     #Name of the file
@@ -79,6 +81,8 @@ The argument must contain:
     attr_reader :crc
     #Path (location) in snapshots
     attr_reader :snapshotpath
+    #File attributes
+    attr_reader :attributes
 
     def to_hash()
       hash = { 
@@ -118,8 +122,7 @@ ID_FILE (0x02)
       else
         bcss << 2
       end
-      bcss << @filename.size
-      bcss << @filename
+      bcss << shortstring2bcss(@filename)
       bcss << fixnum2int64(@timestamp.time2ad)
       #DOS Attributes
       bcss << @attributes  #little endian storage
@@ -169,7 +172,7 @@ FILE_EX_LINK_PATH (0x03)
 
 =end
     def bcss_extended_headers()
-      ehead = ''
+      ehead = ''.force_encoding('binary')
       if @version 
         ehead << 1
         ehead << @version.size
@@ -187,13 +190,12 @@ FILE_EX_LINK_PATH (0x03)
         ehead << utfsymlink.size
         ehead << utfsymlink
       end
-      #fixme
   
-      bcss = ''
+      bcss = ''.force_encoding('binary')
       bcss << fixnum2int16(ehead.size + 2) #size including this length
       bcss << ehead
   
-      bcss 
+      bcss
     end
   end #File
 end #BC3

data/lib/bc3/folder.rb

@@ -63,12 +63,12 @@ The argument must contain:
 * dirname
 =end
   def self.new_by_dirname( dirname )
-#fixme: Pattern??
-#fixme: attrib
     $log.debug("Build folder #{dirname} from file system")
       #The timestamp must be 'local time' -> ignore local time zone for generation
     folder = new( dirname, ::File.mtime(dirname) + Time.now.gmt_offset )
+#fixme: attrib
     Dir.chdir(dirname){
+#feature idea: Define Pattern as method parameter??
       Dir['*'].each{|f|
         if ::File.directory?(f)
           folder << Folder.new_by_dirname(f)
@@ -90,6 +90,10 @@ The argument must contain:
   #timestamp
   attr_reader :timestamp
   attr_reader :attributes
+  #Part of the Extended Header Subtypes, see BC3::SnapshotParser#parse_extended_header_subtypes
+  attr_accessor :ex_utf
+  #Part of the Extended Header Subtypes, see BC3::SnapshotParser#parse_extended_header_subtypes
+  attr_accessor :ex_link_path
 
 =begin rdoc
 Add content to folder.
@@ -175,11 +179,13 @@ Options may be
       options = [:folders, :files ] if options.empty? #take all, but not recursive
       
       extract = {}
-      @content.each{|key, content| 
+      #~ @content.each{|key, content| 
+      #~ @content.sort_by{|key, x| x.is_a?(Folder) ? 'a' : 'b' + x.basename }.each{|key, content| 
+      @content.sort_by{|key, x| ( x.is_a?(Folder) ? 'a%s/' : 'b%s' ) % x.basename }.each{|key, content| 
         case content
           when File
             extract[key] = content if options.include?(:files)
-          when Folder
+      when Folder
             extract[key+'/'] = content if options.include?(:folders)
           else
             raise "Internal error"
@@ -198,8 +204,8 @@ Options may be
         extract
       end
     end
-    
-    #~ def values();@content.values;end  #fixme
+=begin rdoc
+=end
     def inspect()
       "<BC3::Folder #{@dirname}>"
     end
@@ -207,12 +213,14 @@ Options may be
 Return the content in a hash.
 =end
     def to_hash()
-      #fixme: Array with needed keys?
+      #feature idea: Optional array with needed keys?
       { 
         dirname: @dirname,
         timestamp: @timestamp, 
         attributes: @attributes,
-        content: @content.values.map{|x| x.to_hash } 
+        #use each to get a sorted output
+        #~ content: @content.values.map{|x| x.to_hash } 
+        content: self.each.values.map{|x| x.to_hash } 
       }
     end
 =begin rdoc
@@ -234,8 +242,7 @@ ID_DIRECTORY_END (0xFF)
     def bcss() 
       bcss = "".force_encoding('BINARY')
       bcss << 1
-      bcss << @dirname.size
-      bcss << @dirname
+      bcss << shortstring2bcss(@dirname)
       bcss << fixnum2int64(@timestamp.time2ad)
       #DOS Attributes
       bcss << @attributes
@@ -248,6 +255,25 @@ ID_DIRECTORY_END (0xFF)
       }
       
       bcss << 255
+      if ex_utf
+        $log.fatal("Untested ex_utf")
+        bcss << 'xxxx'
+        bcss << 4 #Extended Header Subtypes
+=begin
+EX_UTF8 (0x01)
+    UTF-8 encoded filename for the ID_DIRECTORY that immediately preceeded
+this header.  The length is given in the ID_EXTENDED header and the data is a
+char[].
+    If the .bcss header flags indicate that the data is not UTF-8 and the
+source path is included this can be included as the first record in the file
+in order to give a UTF-8 version of the source path.
+=end        
+        bcss << 1
+        bcss << ex_utf.bytesize
+        #~ p ex_utf.bytesize
+        #~ p ex_utf
+        bcss << shortstring2bcss(ex_utf.dup.force_encoding('binary'))
+      end
       bcss
     end
   end #Folder

data/lib/bc3/helper.rb

@@ -21,7 +21,7 @@ http://www.xxcopy.com/xxcopy06.htm
   0 1 0 0 0 0 0 0	40h	Reserved
   1 0 0 0 0 0 0 0	80h	Reserved
 
-Usage:
+Usage
   attrib = Attrib::ReadOnly | Attrib::Hidden
 =end
 module Attrib
@@ -31,6 +31,30 @@ module Attrib
   VolumeLabel = 8
   Directory = 16
   Archive = 32
+=begin
+How to read attributs/windows:
+http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/68298
+
+Easier: call attrib and analyse output
+
+fixme: CHeck OS
+=end
+  def self.fileattrib(filename)
+    attrib = 0
+    
+    raise ArgumentError, "Filename #{filename} not found" unless ::File.exists?(filename)
+    
+    attrib += Directory if ::File.directory?(filename)
+    res = `attrib #{filename}`
+    
+    attrib += ReadOnly if res =~ /\A\S*R/
+    attrib += Hidden  if res =~ /\A\S*H/
+    attrib += System if res =~ /\A\S*S/
+    #~ attrib += VolumeLabel = if res =~ /\A\S*/
+    attrib += Archive if res =~ /\A\S*A/
+    
+    attrib
+  end
 end
 
 =begin rdoc
@@ -81,6 +105,7 @@ AD's epoch (Bignum) as an bit sequence.
     ('%064b' % int).scan(/(\d{8})/).flatten.reverse.each{|b|
         bindata << b.to_i(2)
     }
+    raise ArgumentError unless $~.post_match.empty?
     raise ArgumentError unless bindata.size == 8  #int was too big
     bindata
   end
@@ -93,6 +118,7 @@ Same as Helper#fixnum2int64, but as 4 bytes string.
     ('%032b' % int).scan(/(\d{8})/).flatten.reverse.each{|b|
         bindata << b.to_i(2)
     }
+    raise ArgumentError unless $~.post_match.empty?
     raise ArgumentError unless bindata.size == 4  #int was too big
     bindata
   end
@@ -105,9 +131,30 @@ Same as Helper#fixnum2int64, but as 2 bytes string.
     ('%016b' % int).scan(/(\d{8})/).flatten.reverse.each{|b|
         bindata << b.to_i(2)
     }
+    raise ArgumentError unless $~.post_match.empty?
     raise ArgumentError unless bindata.size == 2 #int was too big
     bindata
   end
 
+=begin rdoc
+Save a shortstring to bcss
+
+ShortString:
+    Variable length single-byte character string (ANSI or UTF-8).  
+    Not null terminated.
+
+    UTF-8: If set the snapshot was compressed on a system where the default
+character encoding is UTF-8 (Linux, OS X).  Filenames, paths, and link targets
+will all be stored as UTF-8.  If this isn't set the paths are stored using the
+original OS's ANSI codepage (Windows).  In that case any paths may be stored a
+second time as UTF-8 in extended headers.
+
+=end
+  def shortstring2bcss(shortstring)
+      bcss = "".force_encoding('BINARY')
+      bcss << shortstring.bytesize
+      bcss << shortstring.dup.force_encoding('binary')
+      bcss
+  end
 end #Helper
 end #BC3

data/lib/bc3/parse.rb

@@ -3,10 +3,9 @@ Parser for BC3-Snapshots.
 
 The snapshot may be compressed or uncompressed.
 
-No support for UTF-8.
+Test see examples/test_reconstruct.
+
 =end
-$:.unshift('..') if $0 == __FILE__  #only test
-require 'bc3'
 
 require "zlib"
 module BC3
@@ -67,19 +66,20 @@ Parser for a given bcss-file.
       @compressed = rawdata[16].getbyte(0) & 1 != 0
       @sourcepath = rawdata[16].getbyte(0) & 2 != 0
       @reserved    = rawdata[16].getbyte(0) & 4 != 0
-      @utf           = rawdata[16].getbyte(0) & 8 != 0
+      @utf           = rawdata[16].getbyte(0) & 8 != 0      
+      @log.warn("UTF-data") if @utf
       if rawdata[17] != "\x0"
         @log.warn("2nd flag byte is filled")
       end
       
       #Analyse Source path
-      
       #Delete second length parameter for source path
       if rawdata.slice!(19) != "\x0"
         @log.warn("Path > 255 not supported")
         raise "Path > 255 not supported"
       end
       path, body = parse_shortstring(rawdata[18..-1])
+      path.force_encoding('utf-8') if @utf
       if @compressed
 =begin
 Flags:
@@ -101,9 +101,10 @@ Code from http://www.ruby-forum.com/topic/136825
       end
       
       @snapshot = Snapshot.new(path, @timestamp)
-      
+      @snapshot.utf = @utf
+
       parse_body(body)
-    end #initialize
+    end #read_bcss
     #Snapshot-object, result of the parsing.
     attr_reader :snapshot
     attr_reader :timestamp
@@ -193,6 +194,16 @@ ID_FILE_EX (0x03)
               }.merge(extradata)
             )
 =begin
+ID_EXTENDED (0x04)
+    Extended headers
+
+    SubType        : UByte
+    Length         : UWord
+    Data           : Byte[Length]
+=end
+          when "\x04" #file
+            parse_extended_header_subtypes(body, folderstack.last)
+=begin
 ID_DIRECTORY_END (0xFF)
     Represents the end of a directory listing.  No data.
 =end            
@@ -205,8 +216,7 @@ ID_DIRECTORY_END (0xFF)
           end
         end
         if folderstack.size > 1
-          @log.error("Folders in Folderstack not closed correct")
-          p folderstack.size
+          @log.error("Folders in Folderstack not closed correct - #{folderstack.size} levels open")
         end
     end
 =begin rdoc
@@ -216,12 +226,17 @@ Get a "shortstring".
 
 Return shortstring and rest of string.
 =end
-    def parse_shortstring( string )
+    def parse_shortstring( string )    
       #Get length of path
       pathsize = string.slice!(0).bytes.first
                       # + rawdata[19].bytes.first * 255  #--test it 
-      return [string.slice!(0,pathsize), string]
+      if @utf
+        return [string.slice!(0,pathsize).force_encoding('utf-8'), string]
+      else
+        return [string.slice!(0,pathsize), string]
+      end
     end
+
 =begin rdoc
 Get a "longstring".
 
@@ -236,6 +251,11 @@ Return longstring and rest of string.
         @log.warn("longstring > 255 not supported")
         raise "longstring > 255 not supported"
       end
+      if @utf
+        return [string.slice!(0,stringsize).force_encoding('utf-8'), string]
+      else
+        return [string.slice!(0,stringsize), string]
+      end
       return [string.slice!(0,stringsize), string]
     end
 =begin rdoc
@@ -300,7 +320,7 @@ FILE_EX_LINK_PATH (0x03)
       #loop on extradata...
       while !extradata_string.empty?
         case flag = extradata_string.slice!(0)
-          when "\x01"
+          when "\x01" #version
             extradata[:version], extradata_string = parse_shortstring( extradata_string )
           when "\x02" #UTF-8 encoded filename
               extradata[:utfpath], extradata_string = parse_shortstring( extradata_string )
@@ -319,19 +339,76 @@ FILE_EX_LINK_PATH (0x03)
         #~ end
       extradata
     end
+=begin rdoc
+  ========================
+  Extended Header Subtypes
+  ========================
+
+  Extended headers should be written in ascending numeric order.  Once BC sees
+  an extended subtype that it doesn't undertand it stops processing ID_EXTENDED
+  headers until it finds one of ID_DIRECTORY/ID_DIRECTORY_END/ID_FILE/ID_FILE_EX.
+
+Side effect: the parameter will be shortened.
+=end
+    def parse_extended_header_subtypes(data, last_folder)
+        continue = true
+        while continue
+          case flag = data.slice!(0)
+=begin
+EX_UTF8 (0x01)
+    UTF-8 encoded filename for the ID_DIRECTORY that immediately preceeded
+this header.  The length is given in the ID_EXTENDED header and the data is a
+char[].
+    If the .bcss header flags indicate that the data is not UTF-8 and the
+source path is included this can be included as the first record in the file
+in order to give a UTF-8 version of the source path.
+=end
+            when "\x01"
+              ex_utf, data = parse_shortstring( data )
+              last_folder.ex_utf = ex_utf.force_encoding('utf-8')
+=begin
+EX_DIRECTORY_EX (0x02)
+    Extended directory header for the ID_DIRECTORY that immediately preceeded
+this header.  Data is the record below, but Length may be larger to support
+future expansion.
+
+    Flags         : UByte
+      Bit : Meaning
+        0 : Error - Contents not available.  Flag as a load error in BC.
+=end
+          when "\x02" #UTF-8 encoded filename
+              @log.fatal("Undefined extended_header_subtypes 2")
+              ex_directory_ex = data.slice!(0)
+=begin
+EX_RESYNC (0x03)
+    Works around a bug in Beyond Compare's parser in versions prior to 3.2.2.
+If an ID_DIRECTORY is followed by any ID_EXTENDED headers besides EX_UTF8 or
+EX_DIRECTORY_EX include one copy of this header before them.
+
+    Length : UWord   = 0x0001
+    Data   : Byte[1] = 0
+=end
+          when "\x03" #UTF-8 encoded symbolic link path
+              @log.fatal("Undefined extended_header_subtypes 3")
+              size = data.slice!(0).bytes.first
+              ex_resync = data.slice!(size)
+=begin
+EX_LINK_PATH (0x04)
+    UTF-8 encoded symbolic link path for the ID_DIRECTORY that immediately
+preceeded this header.  The length is given in the ID_EXTENDED header and the
+data is a char[].
+=end
+          when "\x04" #UTF-8 encoded symbolic link path
+              @log.fatal("Undefined extended_header_subtypes 4")
+              ex_link_path, data = parse_shortstring( data )
+              last_folder.ex_link_path = ex_link_path.force_encoding('utf-8')
+          else
+              @log.debug("Stop extended header subtype handling #{flag.inspect} <#{data.inspect}>")
+              continue = false #stop evaluation
+            end
+          end #while continue
+    end #parse_extended_header_subtypes
+    
   end #SnapshotParser
   def to_hash; @snapshot.to_hash; end
 end
-
-if $0 == __FILE__
-require 'yaml'
-#~ x = BC3::SnapshotParser.new('../../examples/results/testdir_2011-01-16.bcssx' )
-#~ x = BC3::SnapshotParser.new('../../examples/results/bc3_2011-01-16.bcss' )
-x = BC3::SnapshotParser.new('../../Uncompressed Sample/Uncompressed Sample.bcss' )
-#~ puts x.snapshot.to_hash.to_yaml
-x.snapshot.save('../../Uncompressed Sample/Uncompressed Sample_reconstructed.bcss')
-FileUtils.copy(
-  '../../Uncompressed Sample/Uncompressed Sample_reconstructed.bcss',
-  '../../Uncompressed Sample/Uncompressed Sample_reconstructed.xxxx'
-  )
-end

data/lib/bc3/snapshot.rb

@@ -49,6 +49,8 @@ A snapsot-hash must contain:
     raise ArgumentError, "Content is no array" unless data[:content].is_a?(Array)
     
     snapshot = new( data[:snapshot], data[:timestamp] )
+    snapshot.utf = data[:utf]
+    
     data[:content].each{| element |
       if element.has_key?(:dirname)
         snapshot << Folder.newh(element)
@@ -114,18 +116,67 @@ Empty folders end with a slash.
     snap
   end #self.new_filelist
 
-  #Keep old definition
+  #Keep old definition, version 0.1.0
   class << self
     alias :newd :new_filesystem
     alias :newh :new_hash
   end
 
+=begin rdoc
+Uncompress a snapshot and save it again.
+=end
+  def self.uncompress(filename, filename2 = nil)
+    rawdata = nil
+    ::File.open( filename, 'rb' ){|f| rawdata = f.read() }
+    
+    compressed = rawdata[16].getbyte(0) & 1 != 0
+    
+    pathlength = rawdata.slice(18).bytes.first
+    if rawdata.slice(19) != "\x0"
+      @log.warn("Path > 255 not supported")
+      raise "Path > 255 not supported"
+    end
+    path = rawdata[20,pathlength]
+
+    header = rawdata[0,16]
+    if compressed
+      header << (rawdata.slice(16).bytes.first - 1)
+    else
+      header << rawdata.slice(16)
+    end
+    header << rawdata[17,3] #keep original length
+    #~ header << path.bytesize #byte 18
+    #~ header << 0 #byte 19 - for long pathes
+    header << path
+
+    #will change again for compressed data
+    body = rawdata[20 + pathlength ..-1]
+    if compressed
+      begin
+        body= Zlib::Inflate.inflate(body);  #Unclear problem
+      rescue Zlib::DataError
+        $log.debug("Zlib::DataError occured - try with raw  deflate")
+        #no luck with Zlib decompression. Let's try with raw  deflate,
+        #like some broken browsers do.
+        body= Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(body)
+      end
+    end
+    
+    ::File.open( filename2, 'wb' ){|f|
+      f << header
+      f << body
+    } if filename2
+    header+body
+  end
+
   #homepath of the snapshot
   attr_reader :path
   #Content of the snapshot
   attr_reader :basefolder
   #Time stamp from snapshot. Default 'now'
   attr_reader :timestamp
+  #UTF-Flag. Is set by SnapshotParser#read_bcss
+  attr_accessor :utf
   
 =begin rdoc
 Add content (folders/files) to snapshot.
@@ -265,11 +316,14 @@ Usefull in combination with yaml:
   puts snapshot.to_hash.to_yaml
 =end
     def to_hash()
-      {
-        snapshot: @path,
-        timestamp: @timestamp,
-        content: @basefolder.each.values.map{| x | x.to_hash }
-      }
+      result =   {
+                      snapshot: @path,
+                      timestamp: @timestamp,
+                      content: @basefolder.each.values.map{| x | x.to_hash }
+                    }
+      result[:utf] = @utf unless @utf.nil?
+
+      result
     end
 =begin rdoc
 Prepare a snapshot (bcss-file).
@@ -380,24 +434,25 @@ second time as UTF-8 in extended headers.
       
       # [10..11] = Flags         (UWord)
 
-            #~ Bit : Meaning
-              #~ 0 : Compressed
-              #~ 1 : Source Path included
-              #~ 2 : Reserved
-              #~ 3 : UTF-8
+            #~ Bit/Value : Meaning
+              #~ 0/1 : Compressed
+              #~ 1/2 : Source Path included
+              #~ 2/4 : Reserved
+              #~ 3/8 : UTF-8
            #~ 4-15 : Reserved
       flag = 0  #no flag set
       flag += 2 #Source Path included
       flag += 1 if compressed
+      flag += 8 if @utf
       header << flag
       header << 0
 
       # [12..13] = Path Length (UWord)   | Optional
-      header << @path.size
+      header << @path.bytesize
       header << 0 #fixme if path > 255
       raise "too long path" if @path.size > 155 #fixme
       # [14..N]  = Path        (char[])  |
-      header << @path
+      header << @path.dup.force_encoding('binary')
       
       header
     end #header

data/lib/bc3/time.rb

@@ -1 +1 @@
-=begin rdoc
  bignum too big to convert into `unsigned long' (RangeError)
  AD_EPOCH      = 116_444_736_000_000_000
  AD_MULTIPLIER = 10_000_000
  # convert a Time object to AD's epoch
  def time2ad()
    (self.to_i * AD_MULTIPLIER) + AD_EPOCH
  end
  # convert from AD's time string to a Time object
  def self.ad2time(time)
    Time.at((time.to_i - AD_EPOCH) / AD_MULTIPLIER)
  end
+=begin rdoc
  bignum too big to convert into `unsigned long' (RangeError)
  AD_EPOCH      = 116_444_736_000_000_000
  AD_MULTIPLIER = 10_000_000
  # convert a Time object to AD's epoch
  def time2ad()
    (self.to_f * AD_MULTIPLIER) + AD_EPOCH
  end
  # convert from AD's time string to a Time object
  def self.ad2time(time)
    Time.at((time.to_f - AD_EPOCH) / AD_MULTIPLIER)
  end

data/unittest/unittest_bc3_file.rb

@@ -30,6 +30,19 @@ class Test_file < Test::Unit::TestCase
     assert_equal('x', file.basename )
     assert_equal(file.filename, file.basename )
   end  
+  def test_new_by_filename()
+    file = BC3::File.new_by_filename( __FILE__)
+    assert_equal(__FILE__, file.filename )
+    assert_equal(__FILE__, file.basename )
+    assert_equal(file.filename, file.basename )
+    assert_kind_of(Integer, file.crc )
+    ##Sure?
+    assert_equal(BC3::Attrib::Archive, file.attributes )
+    
+    #Test without crc
+    file = BC3::File.new_by_filename( __FILE__, false)
+    assert_equal(nil, file.crc )
+  end  
   def test_hash()
     assert_equal(   { filename: 'f', filesize: 5, timestamp: $timestamp, attributes: BC3::Attrib::Archive, crc: nil},
         BC3::File.new( filename: 'f', filesize: 5, timestamp: $timestamp ).to_hash

data/unittest/unittest_bc3_folder.rb

@@ -192,14 +192,14 @@ class Test_folder < Test::Unit::TestCase
     assert_equal( %w{dir2/test.txt},folder.each(:recursive,:files).keys )
     
     folder.add_with_path('dir2/subdir', BC3::File.new(filename: 'test.txt', filesize: 5 ))
-    assert_equal( %w{dir2/test.txt dir2/subdir/test.txt},folder.each(:recursive,:files).keys )
+    assert_equal( %w{dir2/subdir/test.txt dir2/test.txt},folder.each(:recursive,:files).keys )
     
     folder.add_with_path('dir4/subdir', BC3::File.new(filename: 'test.txt', filesize: 5 ))
-    assert_equal( %w{dir2/test.txt dir2/subdir/test.txt dir4/subdir/test.txt},folder.each(:recursive,:files).keys )
+    assert_equal( %w{dir2/subdir/test.txt dir2/test.txt dir4/subdir/test.txt},folder.each(:recursive,:files).keys )
     
-    #Added at end - sorting not necessary.
+    #Added at end, but each is sorted.
     folder.add_with_path('dir0', BC3::File.new(filename: 'test.txt', filesize: 5 ))
-    assert_equal( %w{dir2/test.txt dir2/subdir/test.txt dir4/subdir/test.txt dir0/test.txt},folder.each(:recursive,:files).keys )
+    assert_equal( %w{dir0/test.txt dir2/subdir/test.txt dir2/test.txt dir4/subdir/test.txt},folder.each(:recursive,:files).keys )
   end
 end
 

data/unittest/unittest_bc3_helper.rb

@@ -0,0 +1,44 @@
+gem 'test-unit'
+require 'test/unit'
+
+$:.unshift('../lib')
+require 'bc3/helper'
+
+class Test_helper < Test::Unit::TestCase
+  include BC3::Helper
+  def test_fixnum2int64()
+    assert_equal("\x05\x00\x00\x00\x00\x00\x00\x00", fixnum2int64(5))
+    assert_equal("\xff\x00\x00\x00\x00\x00\x00\x00", fixnum2int64(255))
+    assert_equal("\x00\x01\x00\x00\x00\x00\x00\x00", fixnum2int64(256))
+    assert_equal("\x01\x01\x00\x00\x00\x00\x00\x00", fixnum2int64(257))
+    assert_equal("\x00\x04\x00\x00\x00\x00\x00\x00", fixnum2int64(1024))
+    assert_equal("\xff\xff\xff\xff\xff\xff\xff\xff", fixnum2int64( 'ffffffffffffffff'.to_i(16)))
+    
+    assert_raise(ArgumentError){fixnum2int64('ffffffffffffffff'.to_i(16) + 1)}    
+  end
+  def test_fixnum2int32()
+    assert_equal("\x05\x00\x00\x00", fixnum2int32(5))
+    assert_equal("\xff\x00\x00\x00", fixnum2int32(255))
+    assert_equal("\x00\x01\x00\x00", fixnum2int32(256))
+    assert_equal("\x01\x01\x00\x00", fixnum2int32(257))
+    assert_equal("\x00\x04\x00\x00", fixnum2int32(1024))
+    assert_equal("\xff\xff\xff\xff", fixnum2int32( 'ffffffff'.to_i(16)))
+    
+    assert_raise(ArgumentError){fixnum2int32('ffffffff'.to_i(16) + 1)}    
+  end
+  def test_fixnum2int16()
+    assert_equal("\x05\x00", fixnum2int16(5))
+    assert_equal("\xff\x00", fixnum2int16(255))
+    assert_equal("\x00\x01", fixnum2int16(256))
+    assert_equal("\x01\x01", fixnum2int16(257))
+    assert_equal("\x00\x04", fixnum2int16(1024))
+    assert_equal("\xff\xff", fixnum2int16( 'ffff'.to_i(16)))
+    
+    assert_raise(ArgumentError){fixnum2int16('ffff'.to_i(16) + 1)}
+  end
+  def test_crc32()
+    assert_equal('CBF43926', '%X' % BC3::Helper.crc32('123456789'))
+    assert_equal('D87F7E0C', '%X' % BC3::Helper.crc32('test'))
+  end
+end
+

data/unittest/unittest_bc3_merge.rb

@@ -81,6 +81,7 @@ class Test_merge < Test::Unit::TestCase
     merge = { 
       snapshot: 'merge_b',
       timestamp: result[:timestamp],
+      utf: false,
       content: [
         { dirname: 'snap1', timestamp: $timestamp, attributes: 16, content: Snap1[:content] },
         { dirname: 'snap2', timestamp: $timestamp, attributes: 16, content: Snap2[:content] },
@@ -98,6 +99,7 @@ class Test_merge < Test::Unit::TestCase
     merge = { 
       snapshot: 'merge_i',
       timestamp: result[:timestamp],
+      utf: false,
       content: Snap1[:content] + Snap2[:content]
       }
     assert_equal(merge, result)

data/unittest/unittest_bc3_snapshot.rb

@@ -64,8 +64,9 @@ class Test_snapshot < Test::Unit::TestCase
     
     hash[:content] << { filename: 'file2', filesize: 15, timestamp: $timestamp, crc: nil, attributes: BC3::Attrib::Archive, }
     assert_equal(hash, BC3::Snapshot.new_hash( hash ).to_hash)
-    
-    hash[:content] << { dirname: 'dir1', timestamp: $timestamp, :content => [], attributes: BC3::Attrib::Directory, }
+
+    #No <<, sort must be seen
+    hash[:content].unshift({ dirname: 'dir1', timestamp: $timestamp, :content => [], attributes: BC3::Attrib::Directory, })
     assert_equal(hash, BC3::Snapshot.new_hash( hash ).to_hash)
   end
   def test_new_filesystem()
@@ -104,17 +105,17 @@ class Test_snapshot < Test::Unit::TestCase
     )
 
     assert_equal( %w{
-                file
                 folder/
-                folder/file1
                 folder/subfolder/
                 folder/subfolder/subfile1
                 folder/subfolder2/
+                folder/file1
+                file
       }, bc3.each(:files, :folders ).keys)
 
     assert_equal( %w{
-                file
                 folder/
+                file
       }, bc3.each(:rootonly, :files, :folders ).keys)
       
     assert_equal( %w{
@@ -124,9 +125,9 @@ class Test_snapshot < Test::Unit::TestCase
       }, bc3.each(:folders ).keys)
       
     assert_equal( %w{
-                file
-                folder/file1
                 folder/subfolder/subfile1
+                folder/file1
+                file
       }, bc3.each(:files ).keys)
 
 
@@ -220,12 +221,12 @@ class Test_snapshot < Test::Unit::TestCase
 
     #test index
     assert_equal(%w{
-                file
                 folder/
-                folder/file1
                 folder/subfolder/
                 folder/subfolder/subfile1
                 folder/subfolder2/
+                folder/file1
+                file
       }, bc3.index.keys)
   end #test_index
   def test_index_automatic_reset()

data/unittest/unittest_bc3_time.rb

@@ -0,0 +1,95 @@
+=begin rdoc
+There is a problem with nanoseconds.
+-> test_conversion_nanoseconds and test_conversion_now are inactivated.
+=end
+gem 'test-unit'
+require 'test/unit'
+
+$:.unshift('../lib')
+require 'bc3/time'
+
+class Test_time < Test::Unit::TestCase
+  #Nonoseconds are ignored (missing feature in Time-class?)
+  def test_ad2time()
+    assert_equal('2006-08-17 09:19:04 000000000', 
+          Time.ad2time(128002727440000000).strftime("%Y-%m-%d %H:%M:%S %9N")
+          )          
+    assert_equal('2006-08-17 09:18:20 000000000', 
+          Time.ad2time(128002727000000000).strftime("%Y-%m-%d %H:%M:%S %9N")
+          )    
+          
+    assert_match(/2006-08-17 09:19:04 08082\d\d\d\d/,
+          Time.ad2time(128002727440808261).strftime("%Y-%m-%d %H:%M:%S %9N")
+          )
+    #~ assert_equal('2006-08-17 09:19:04 0808261',
+    assert_equal('2006-08-17 09:19:04 080825567', #??
+          Time.ad2time(128002727440808261).strftime("%Y-%m-%d %H:%M:%S %9N")
+          )
+  end
+        
+  def test_time2ad()
+    assert_equal(128002464000000000, Time.utc(2006,8,17).time2ad())
+    assert_equal(128002727440000000, Time.local(2006,8,17,9,19,4).time2ad())
+    
+    #Test does not work. Split in two areas (high/Low) makes calculating unpossible.
+    #~ p 60 * 60 * 1_000_000_000  #one hour in nanoseconds
+    #~ assert_equal( 3_600_000_000_000, #one hour in nanoseconds
+        #~ Time.utc(2006,8,17,12).time2ad() - Time.utc(2006,8,17,11).time2ad()
+    #~ )
+
+    #~ assert_equal( 3_600_000_000_000, #one hour in nanoseconds
+        #~ Time.utc(2006,8,17).time2ad() - Time.local(2006,8,17).time2ad()
+    #~ )
+  end
+  def test_conversions()
+    testtime = Time.utc(2006,8,17)
+    assert_equal(testtime, Time.ad2time(testtime.time2ad))
+    
+    testtime = Time.local(2006,8,17)
+    assert_equal(testtime, Time.ad2time(testtime.time2ad))
+  end
+  
+
+  def test_conversion_ok()
+    
+    #last 7 digits must be 0.
+    last_logon = 129393178970000000
+    
+    time = Time.ad2time(last_logon)
+    last_logon2 = time.time2ad()
+    time2 = Time.ad2time(last_logon2)
+
+    assert_equal(time, time2)
+    
+    assert_equal(0, last_logon - last_logon2)
+    assert_equal(last_logon, last_logon2)    
+  end
+  def test_conversion_nanoseconds()
+    
+    #last 7 digits are nano seconds.
+    last_logon = 129393178978310001
+    time = Time.ad2time(last_logon)
+    
+    last_logon2 = time.time2ad()
+    time2 = Time.ad2time(last_logon2)
+
+    #~ assert_in_delta(0, time - time2, 0.001)
+    assert_equal(0, time - time2)
+    assert_equal(time, time2)
+    
+    assert_equal(0, last_logon - last_logon2)
+    assert_equal(last_logon, last_logon2)
+    
+  end
+  #
+  def test_conversion_now()
+    testtime = Time.now
+    #problem: 
+    #AD-time is "the number of 100-nanosecond intervals since ..."
+    #Time.now is nanoseconds
+    assert_in_delta(0, testtime - Time.ad2time(testtime.time2ad), 1E-6)
+    #~ assert_equal(0, testtime - Time.ad2time(testtime.time2ad))
+    #~ assert_equal(testtime, Time.ad2time(testtime.time2ad))
+  end
+end
+

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: bc3
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 31
   prerelease: false
   segments: 
   - 0
   - 1
-  - 1
-  version: 0.1.1
+  - 2
+  version: 0.1.2
 platform: ruby
 authors: 
 - Knut Lickert
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-30 00:00:00 +01:00
+date: 2011-02-20 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -47,9 +47,11 @@ extensions: []
 
 extra_rdoc_files: 
 - BCSS_Binary_Format.txt
+- changes.rd
 - bin/bc3_merge.rb
 - bin/bc3_search.rb
 files: 
+- changes.rd
 - BCSS_Binary_Format.txt
 - bin/bc3_merge.rb
 - bin/bc3_search.rb
@@ -60,19 +62,28 @@ files:
 - lib/bc3/parse.rb
 - lib/bc3/snapshot.rb
 - lib/bc3/time.rb
-- unittest/unittest_bc3.rb
-- unittest/unittest_bc3_file.rb
-- unittest/unittest_bc3_folder.rb
-- unittest/unittest_bc3_snapshot.rb
-- unittest/unittest_bc3_merge.rb
-- unittest/unittest_bc3_search.rb
 - examples/test_combine.rb
 - examples/test_filesystem.rb
 - examples/test_hardcoded.rb
 - examples/test_yaml.rb
+- examples/test_reconstruct.rb
 - examples/test_merge.bat
 - examples/folder1_2011-01-21.bcss
 - examples/folder2_2011-01-21.bcss
+- examples/results/readme
+- examples/scootersoftware/readme
+- examples/scootersoftware/UncompressedSample.bcss
+- examples/scootersoftware/UnicodeFilenames(Linux).bcss
+- examples/scootersoftware/UnicodePath(Linux).bcss
+- examples/scootersoftware/UnicodeFilenames(Win32).bcss
+- examples/scootersoftware/UnicodePath(Win32).bcss
+- unittest/unittest_bc3_helper.rb
+- unittest/unittest_bc3_time.rb
+- unittest/unittest_bc3_file.rb
+- unittest/unittest_bc3_folder.rb
+- unittest/unittest_bc3_snapshot.rb
+- unittest/unittest_bc3_merge.rb
+- unittest/unittest_bc3_search.rb
 has_rdoc: true
 homepage: http://gems.rubypla.net/bc3
 licenses: []
@@ -109,16 +120,10 @@ signing_key:
 specification_version: 3
 summary: Build and analyse Beyond Compare (BC3) Snapshot-Files.
 test_files: 
-- unittest/unittest_bc3.rb
+- unittest/unittest_bc3_helper.rb
+- unittest/unittest_bc3_time.rb
 - unittest/unittest_bc3_file.rb
 - unittest/unittest_bc3_folder.rb
 - unittest/unittest_bc3_snapshot.rb
 - unittest/unittest_bc3_merge.rb
 - unittest/unittest_bc3_search.rb
-- examples/test_combine.rb
-- examples/test_filesystem.rb
-- examples/test_hardcoded.rb
-- examples/test_yaml.rb
-- examples/test_merge.bat
-- examples/folder1_2011-01-21.bcss
-- examples/folder2_2011-01-21.bcss

data/unittest/unittest_bc3.rb

@@ -1,66 +0,0 @@
-gem 'test-unit'
-require 'test/unit'
-
-$:.unshift('../lib')
-require 'bc3'
-
-class Test_helper < Test::Unit::TestCase
-  include BC3::Helper
-  def test_fixnum2int64()
-    assert_equal("\x05\x00\x00\x00\x00\x00\x00\x00", fixnum2int64(5))
-    assert_equal("\xff\x00\x00\x00\x00\x00\x00\x00", fixnum2int64(255))
-    assert_equal("\x00\x01\x00\x00\x00\x00\x00\x00", fixnum2int64(256))
-    assert_equal("\x01\x01\x00\x00\x00\x00\x00\x00", fixnum2int64(257))
-    assert_equal("\x00\x04\x00\x00\x00\x00\x00\x00", fixnum2int64(1024))
-  end
-  def test_fixnum2int32()
-    assert_equal("\x05\x00\x00\x00", fixnum2int32(5))
-    assert_equal("\xff\x00\x00\x00", fixnum2int32(255))
-    assert_equal("\x00\x01\x00\x00", fixnum2int32(256))
-    assert_equal("\x01\x01\x00\x00", fixnum2int32(257))
-    assert_equal("\x00\x04\x00\x00", fixnum2int32(1024))
-  end
-  def test_crc32()
-    assert_equal('CBF43926', '%X' % BC3::Helper.crc32('123456789'))
-    assert_equal('D87F7E0C', '%X' % BC3::Helper.crc32('test'))
-  end
-end
-
-
-class Test_time < Test::Unit::TestCase
-  #Nonoseconds are ignored (missing feature in Time-class?)
-  def test_ad2time()
-    assert_equal('2006-08-17 09:19:04 000000000', 
-          Time.ad2time(128002727440808261).strftime("%Y-%m-%d %H:%M:%S %9N")
-          )
-    assert_equal('2006-08-17 09:19:04 000000000', 
-          Time.ad2time(128002727440000000).strftime("%Y-%m-%d %H:%M:%S %9N")
-          )
-  end
-  def test_time2ad()
-    assert_equal(128002464000000000, Time.utc(2006,8,17).time2ad())
-    assert_equal(128002727440000000, Time.local(2006,8,17,9,19,4).time2ad())
-    
-    #Test does not work. Split in two areas (high/Low) makes calculating unpossible.
-    #~ p 60 * 60 * 1_000_000_000  #one hour in nanoseconds
-    #~ assert_equal( 3_600_000_000_000, #one hour in nanoseconds
-        #~ Time.utc(2006,8,17,12).time2ad() - Time.utc(2006,8,17,11).time2ad()
-    #~ )
-
-    #~ assert_equal( 3_600_000_000_000, #one hour in nanoseconds
-        #~ Time.utc(2006,8,17).time2ad() - Time.local(2006,8,17).time2ad()
-    #~ )
-  end
-  def test_conversions()
-    testtime = Time.utc(2006,8,17)
-    assert_equal(testtime, Time.ad2time(testtime.time2ad))
-    
-    testtime = Time.local(2006,8,17)
-    assert_equal(testtime, Time.ad2time(testtime.time2ad))
-    
-    #Test with now does not work. Changed nanoseconds?
-    #~ testtime = Time.now
-    #~ assert_equal(testtime, Time.ad2time(testtime.time2ad))
-  end
-end
-