bc3 0.1.0 → 0.1.1

data/lib/bc3.rb

@@ -34,15 +34,13 @@ Prepare a virtual folder structure as you would like to have
 and use BC3 to convert your real structure to your wishes.
 
 ==Scan for identic files.
-<em>
-The only way to find duplicate files is to flatten your folder structure so that all the files appear to be at the same level. You then can enable/show the CRC column, sort by it, then manually scan the file list for files with the same size and CRC code. I wouldn't advise it for a large number of files and folders, though. It could take a while to calculate all the CRC values, and it would be a tedious process to manually look through them all for dups.
-</em>
-http://www.scootersoftware.com/vbulletin/showpost.php?p=27736&postcount=4
+See also bin/bc3_search.rb
 
 With this gem, you can analyse a snapshot, copy the data in a hash
 and search for duplicate CRCs
 
 
+
 =Warning and Restrictions
 Please don't 
   require 'bc3'
@@ -69,8 +67,10 @@ Maximum 2GB (more is supported by BC3, but not by this gem).
 =end
 require 'date'
 require 'zlib'
+require 'yaml'
 
 require 'log4r'
+
 $log = Log4r::Logger.new('BC3')
 $log.outputters = Log4r::StdoutOutputter.new('log_stdout')
 $log.level = Log4r::INFO
@@ -85,5 +85,5 @@ require 'bc3/time'
 require 'bc3/parse'
 
 module BC3
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
 end

data/lib/bc3/file.rb

@@ -13,7 +13,7 @@ If you need the original File inside BC3-module, use
     KEYS = [:filename,  :filesize ]
     KEYS_OPTIONAL = [
         :timestamp, :crc, :attributes,
-        :version,
+        :version, :utfpath, :utfsymlink,
       ]
     
 =begin rdoc
@@ -23,7 +23,7 @@ Arguments are given in a hash.
 Must contain KEYS and supports KEYS_OPTIONAL
 =end
     def initialize( args )
-      raise ArgumentError unless args.kind_of?(Hash)
+      raise ArgumentError, "No hash given (#{args.inspect})" unless args.kind_of?(Hash)
       KEYS.each{|key|
         raise ArgumentError, "Missing Key #{key}" unless args.has_key?(key)
       }
@@ -38,7 +38,10 @@ Must contain KEYS and supports KEYS_OPTIONAL
       @attributes = args[:attributes] || Attrib::Archive
       
       @version = args[:version]  #-> Extended File
+      @utfpath = args[:utfpath]  #-> Extended File
+      @utfsymlink = args[:utfsymlink]  #-> Extended File
       
+      @snapshotpath = {}
       #Test content
       raise ArgumentError, "timestamp is no time-object" unless @timestamp.kind_of?(Time)
     end
@@ -52,7 +55,7 @@ The argument must contain:
     def self.new_by_filename( filename )
       $log.debug("Build file #{filename} from file system")
       
-      #fixme: attrib
+      #fixme: get attributes from file system
       #~ p ::File.stat(filename)
       #~ Get attributes: http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/68298
       
@@ -69,16 +72,27 @@ The argument must contain:
       } #fixme crc if ...
       return File.new( settings )
     end
+    #Name of the file
     attr_reader :filename
+    #Name of the file
+    alias :basename :filename
+    attr_reader :crc
+    #Path (location) in snapshots
+    attr_reader :snapshotpath
 
     def to_hash()
-      { 
+      hash = { 
         filename: @filename,
         filesize: @filesize,
         crc: @crc,
         attributes: @attributes,
-        timestamp: @timestamp
+        timestamp: @timestamp,
       }
+      hash[:version] = @version if @version
+      hash[:utfpath] = @utfpath if @utfpath
+      hash[:utfsymlink] = @utfsymlink if @utfsymlink
+
+      hash
     end
     def inspect()
       "<BC3::File #{@filename}>"
@@ -99,7 +113,11 @@ ID_FILE (0x02)
 =end
     def bcss() 
       bcss = "".force_encoding('BINARY')
-      bcss << 2
+      if @version or @utfpath or @utfsymlink
+        bcss << 3 #with file extended_headers
+      else
+        bcss << 2
+      end
       bcss << @filename.size
       bcss << @filename
       bcss << fixnum2int64(@timestamp.time2ad)
@@ -113,8 +131,69 @@ ID_FILE (0x02)
       
       #CRC32
       bcss << fixnum2int32(@crc || 0)
-      #~ bcss << 255      
+      #extended header
+      bcss << bcss_extended_headers if @version or @utfpath or @utfsymlink
+
       bcss
     end
-  end #Folder
+=begin rdoc
+
+ID_FILE (0x02) +
+    ExtraLen       : UInt16
+    ExtraData      : Byte[ExtraLen]
+
+=====================
+File Extended Headers
+=====================
+
+Like extended headers, file extended headers should be written in ascending
+numeric order.  Multiple headers may occur within a single ID_FILE_EX record,
+and compliant parsers should break once they read a type they don't recognize.
+
+FILE_EX_VERSION (0x01)
+    String representation of an executable file's Major/Minor/Maint/Build
+version (e.g., "2.11.28.3542").
+
+    Length : UByte
+    Data   : char[Length]
+
+
+FILE_EX_UTF8 (0x02)
+    UTF-8 encoded filename.  Stored as a FileExString.  Only used if the UTF-8
+name doesn't match the ANSI encoded one or if the filename is longer than 255
+characters.
+
+
+FILE_EX_LINK_PATH (0x03)
+    UTF-8 encoded symbolic link path.  Stored as a FileExString.
+
+=end
+    def bcss_extended_headers()
+      ehead = ''
+      if @version 
+        ehead << 1
+        ehead << @version.size
+        ehead << @version
+      end
+      if @utfpath
+        ehead << 2
+        utfpath = @utfpath.dup.force_encoding('binary')
+        ehead << utfpath.size
+        ehead << utfpath
+      end
+      if @utfsymlink
+        ehead << 3
+        utfsymlink = @utfsymlink.dup.force_encoding('binary')
+        ehead << utfsymlink.size
+        ehead << utfsymlink
+      end
+      #fixme
+  
+      bcss = ''
+      bcss << fixnum2int16(ehead.size + 2) #size including this length
+      bcss << ehead
+  
+      bcss 
+    end
+  end #File
 end #BC3

data/lib/bc3/folder.rb

@@ -10,7 +10,7 @@ module BC3
     def initialize( dirname, timestamp = Time.now, attributes = nil )
       @dirname = dirname
       $log.debug("Create folder #{@dirname}")
-      raise ArgumentError unless @dirname.is_a?(String)
+      raise ArgumentError, "Dirname no string #{@dirname.inspect}"  unless @dirname.is_a?(String)
       
       @timestamp = timestamp || Time.now
       raise ArgumentError, "No Timestamp for Folder" unless @timestamp.is_a?(Time)
@@ -18,6 +18,8 @@ module BC3
       @attributes = attributes || Attrib::Directory
       
       @content = {}
+      @snapshotpath = {}
+      
     end
 =begin rdoc
 Create a folder from a hash.
@@ -81,42 +83,84 @@ The argument must contain:
   end
   #Name of the folder
   attr_reader :dirname
+  #Name of the folder
+  alias :basename :dirname
+  #Path (location) in snapshots
+  attr_reader :snapshotpath
+  #timestamp
+  attr_reader :timestamp
+  attr_reader :attributes
+
 =begin rdoc
-Add content to Folder.
+Add content to folder.
 The 'content' must support the bcss-method.
 
 Content may be:
-* another folder
+* another folder (if pathes are similar to existing folder: see #mixin)
 * a file
+
+If another object with similar name exists, it will be overwritten.
 =end
     def << (content)
-      if !content.respond_to?( :bcss )
+      #Check for BC3::File or BC3::Folder
+      if ! ( content.respond_to?( :bcss ) and content.respond_to?( :basename ) )
         $log.error("Add unknown datatype as content #{content.inspect}")
         raise ArgumentError, "Add unknown datatype as content #{content.inspect}"
       end
       
-      key = content.respond_to?(:dirname) ? content.dirname : content.filename
-      if @content[key]  #content already available
-        if @content[key].respond_to? :dirname and content.respond_to? :dirname
-          $log.debug("Mix into folder #{key} in folder #{self.inspect}")
-          @content[key].mixin(content) 
+      #reset index in relatd snapshot.
+      self.snapshotpath.each{|snapshot, path| snapshot.reset_index }
+  
+      if @content[content.basename]  #content already available
+        #Target and new element are both folders?
+        if @content[content.basename].respond_to? :dirname and content.respond_to? :dirname
+          $log.debug("Mix into folder #{content.basename} in folder #{self.inspect}")
+          #return the folder, where it is mixed in
+          content = @content[content.basename].mixin(content)
         else
-          $log.warn("Overwrite #{key} in folder #{self.inspect}")
-          @content[key] = content
+          $log.info("Overwrite #{content.basename} in folder #{self.inspect}")
+          @content[content.basename] = content
         end
-      else
-        @content[key] = content
+      else  #add new content
+        @content[content.basename] = content
         $log.debug("Add to folder #{self.inspect}: #{content.inspect}") if $log.debug?
       end
+      content #return the folder, where it is mixed in
     end
 =begin rdoc
 Merge two folders.
 The content of a folder is added to the actual content.
 =end
     def mixin( folder )
-      $log.info("Mix in content to folder #{@dirname} in folder #{self.inspect}")
+      $log.debug("Mix in content to folder #{@dirname} in folder #{self.inspect}")
+      raise ArgumentError, "Add unknown datatype as content #{content.inspect}" unless folder.is_a?(Folder)
       folder.each{|key, content| self << content }
+      self  #return the folder, where it is mixed in
+    end
+=begin rdoc
+Add an element with a given path.
+
+Folders will be created for the complete path if necessary.
+=end
+  def add_with_path(path, element)
+    path = path.split(%r{[\\\/]})  unless path.is_a?(Array)
+    if path.empty?
+      self << element
+    else  #subpath
+      #Search target folder
+      target = nil      
+      @content.each{|name, folder|
+        if name == path.first
+          target = folder
+          break #target found
+        end
+      }
+      self << target = Folder.new(path.first) unless target
+      path.shift 
+      target.add_with_path( path, element )
     end
+    element
+  end
 =begin rdoc
 Loop on content of the folder and return path + object.
 
@@ -127,27 +171,26 @@ Options may be
   The result is not nested, pathes are complete
 =end
     def each(*options)
-      if options.empty?
-        extract = @content  #take all
-      else
-        extract = {}
-        @content.each{|key, content| 
-          case content
-            when File
-              extract[key] = content if options.include?(:files)
-            when Folder
-              extract[key] = content if options.include?(:folders)
-            else
-              raise "Internal error"
-            end
-            
-            if options.include?(:recursive) and content.is_a?(Folder)
-              content.each(*options).each{|subkey, subcontent|
-                extract[[key, subkey].join('/')] = subcontent
-              }
-            end
-        }
-      end
+      
+      options = [:folders, :files ] if options.empty? #take all, but not recursive
+      
+      extract = {}
+      @content.each{|key, content| 
+        case content
+          when File
+            extract[key] = content if options.include?(:files)
+          when Folder
+            extract[key+'/'] = content if options.include?(:folders)
+          else
+            raise "Internal error"
+          end
+          
+          if options.include?(:recursive) and content.is_a?(Folder)
+            content.each(*options).each{|subkey, subcontent|
+              extract[[key, subkey].join('/')] = subcontent
+            }
+          end
+      }
       
       if block_given?
         extract.each{|key, content| yield key, content }
@@ -155,6 +198,7 @@ Options may be
         extract
       end
     end
+    
     #~ def values();@content.values;end  #fixme
     def inspect()
       "<BC3::Folder #{@dirname}>"
@@ -211,12 +255,14 @@ end #BC3
 
 if $0 == __FILE__
   
-    folder = BC3::Folder.new('x')
-    folder << BC3::Folder.new('x')
-    folder << BC3::File.new( filename: 'x2', filesize: 1)
-    folder << BC3::File.new( filename: 'x', filesize: 1)
+    folder = BC3::Folder.new('base')
+    #~ folder << BC3::Folder.new('dir')
+    #~ folder << BC3::File.new( filename: 'x2', filesize: 1)
+    #~ folder << BC3::File.new( filename: 'x', filesize: 1)
+    #~ folder.add_with_path('dir2', BC3::File.new(filename: 'test.txt', filesize: 5 ))
+    folder.add_with_path('dir2/subdir', BC3::File.new(filename: 'test,txt', filesize: 5 ))
     
-    puts folder.each
+    puts folder.to_hash.to_yaml
 exit
   
   require 'YAML'
@@ -233,7 +279,10 @@ exit
             :filesize: 15
       }))
     
-    x = folder.each(:folders,:files, :recursive)
-    puts "=========="
-    puts x.keys
-    end
+    #~ x = folder.each(:folders,:files, :recursive)
+    #~ puts "=========="
+    puts folder.to_hash.to_yaml
+    
+    
+    
+  end

data/lib/bc3/helper.rb

@@ -97,5 +97,17 @@ Same as Helper#fixnum2int64, but as 4 bytes string.
     bindata
   end
   
+=begin rdoc
+Same as Helper#fixnum2int64, but as 2 bytes string.
+=end
+  def fixnum2int16( int )
+    bindata = ''.force_encoding('BINARY')
+    ('%016b' % int).scan(/(\d{8})/).flatten.reverse.each{|b|
+        bindata << b.to_i(2)
+    }
+    raise ArgumentError unless bindata.size == 2 #int was too big
+    bindata
+  end
+
 end #Helper
 end #BC3

data/lib/bc3/parse.rb

@@ -1,7 +1,11 @@
 =begin rdoc
+Parser for BC3-Snapshots.
 
+The snapshot may be compressed or uncompressed.
+
+No support for UTF-8.
 =end
-$:.unshift('..')
+$:.unshift('..') if $0 == __FILE__  #only test
 require 'bc3'
 
 require "zlib"
@@ -15,9 +19,24 @@ Parser for a given bcss-file.
 
 =end
     def initialize( filename )
-      rawdata = nil
       @log = $log #fixme replace with sublogger
       @log.info("Read and parse #{filename}")
+      @timestamp = Time.now
+      case filename
+        when /\.ya?ml/
+          ::File.open(filename ){|f|
+            @snapshot = Snapshot.new_hash(YAML.read(f))
+          }
+        when /\.bcssx?/
+          read_bcss(filename)
+        else
+          raise ArgumentError, "Undefined filetype #{::File.extname(filename)}"
+        end
+    end
+=begin rdoc
+=end
+    def read_bcss(filename)
+      rawdata = nil
       ::File.open( filename, 'rb' ){|f|
         rawdata = f.read()
       }
@@ -43,7 +62,6 @@ Parser for a given bcss-file.
     [14..N]  = Path        (char[])  |
 =end
       #~ header =  rawdata[0..17]
-      @timestamp = Time.now
       @timestamp, tail = parse_filetime(rawdata[8,8])
       #Analyse flags - byte position 16/hex10
       @compressed = rawdata[16].getbyte(0) & 1 != 0
@@ -163,7 +181,7 @@ ID_FILE_EX (0x03)
             extradata, tail = parse_longstring(tail)
             extradata = parse_file_extended_headers(extradata)
             unless extradata #Skip at prob
-              @log.warn("Skip #{filename}")
+              @log.warn("Skip #{filename} because of unsupported extended header")
               next  
             end
             folderstack.last << File.new({
@@ -279,24 +297,27 @@ FILE_EX_LINK_PATH (0x03)
 =end
     def parse_file_extended_headers(extradata_string)
       extradata = {}
-      case flag = extradata_string.slice!(0)
-        when "\x01"
-          extradata[:version] = parse_shortstring( extradata_string )
-        when "\x02"
-            @log.warn("Undefined extra data handling for UTF-8 encoded filename")
-            return false #fixme
-        when "\x03"
-            @log.warn("Undefined extra data handling for UTF-8 encoded symbolic")
-            return false #fixme
-        else
-            #fixme handling extradata_string
-            @log.warn("Undefined extra data handling #{flag.inspect} <#{extradata_string.inspect}>")
-        end
-        unless extradata_string.empty?
-          @log.warn("Undefined extra data handling <#{extradata_string.inspect}>")
-          p extradata_string
-        end
-        extradata
+      #loop on extradata...
+      while !extradata_string.empty?
+        case flag = extradata_string.slice!(0)
+          when "\x01"
+            extradata[:version], extradata_string = parse_shortstring( extradata_string )
+          when "\x02" #UTF-8 encoded filename
+              extradata[:utfpath], extradata_string = parse_shortstring( extradata_string )
+              extradata[:utfpath].force_encoding('utf-8')
+          when "\x03" #UTF-8 encoded symbolic link path
+              extradata[:utfsymlink], extradata_string = parse_shortstring( extradata_string )
+              extradata[:utfsymlink].force_encoding('utf-8')
+          else
+              @log.error("Undefined extra data handling #{flag.inspect} <#{extradata_string.inspect}>")
+              extradata_string = '' #stop evaluation
+          end
+      end
+        #~ unless extradata_string.empty?
+          #~ @log.warn("Undefined extra data handling <#{extradata_string.inspect}>")
+          #~ p extradata_string
+        #~ end
+      extradata
     end
   end #SnapshotParser
   def to_hash; @snapshot.to_hash; end
@@ -305,8 +326,12 @@ end
 if $0 == __FILE__
 require 'yaml'
 #~ x = BC3::SnapshotParser.new('../../examples/results/testdir_2011-01-16.bcssx' )
-x = BC3::SnapshotParser.new('../../examples/results/bc3_2011-01-16.bcss' )
-#~ x = BC3::SnapshotParser.new('../../Uncompressed Sample/Uncompressed Sample.bcss' )
-puts x.snapshot.to_hash.to_yaml
+#~ x = BC3::SnapshotParser.new('../../examples/results/bc3_2011-01-16.bcss' )
+x = BC3::SnapshotParser.new('../../Uncompressed Sample/Uncompressed Sample.bcss' )
+#~ puts x.snapshot.to_hash.to_yaml
 x.snapshot.save('../../Uncompressed Sample/Uncompressed Sample_reconstructed.bcss')
+FileUtils.copy(
+  '../../Uncompressed Sample/Uncompressed Sample_reconstructed.bcss',
+  '../../Uncompressed Sample/Uncompressed Sample_reconstructed.xxxx'
+  )
 end