bc3 0.1.0 → 0.1.1

data/BCSS_Binary_Format.txt

@@ -1,5 +1,9 @@
 -------------------------------------------------------------------------------
 Beyond Compare Snapshot Format                                      Version 1.1
+                                                               January 25, 2011
+Copyright (c) 2001-2011 Scooter Software, Inc.
+This document may be redistributed.
+For newer versions contact support@scootersoftware.com
 -------------------------------------------------------------------------------
 
 Beyond Compare snapshots (.bcss) are binary files containing the file metadata
@@ -9,6 +13,46 @@ record sizes are variable, so there's no way to seek to arbitrary records
 without reading all of the records before it.
 
 
+==========
+Data Types
+==========
+
+All integer values are stored in little-endian format.
+ 
+UByte:
+    Unsigned 8-bit value
+
+UInt16:
+    Unsigned 16-bit value
+
+Int32:
+    Signed 32-bit value
+
+UInt32:
+    Unsigned 32-bit value
+
+Int64:
+    Signed 64-bit value
+
+char[]:
+    Variable length single-byte character array (ANSI or UTF-8).
+
+FileTime:
+    Windows FILETIME structure.  64-bit value representing the number of
+100-nanosecond intervals since January 1, 1601 UTC.  Stored in local time.
+
+ShortString:
+    Variable length single-byte character string (ANSI or UTF-8).   Not null
+terminated.
+
+    Length   : UByte
+    Data     : char[Length]
+
+FileExString:
+    Variable length single-byte character string (UTF-8).  See "File Extended
+Header" section for details.
+
+
 ===========
 File Header
 ===========
@@ -60,44 +104,6 @@ original OS's ANSI codepage (Windows).  In that case any paths may be stored a
 second time as UTF-8 in extended headers.
 
 
-==========
-Data Types
-==========
-   
-UByte:
-    Unsigned 8-bit value
-
-UInt16:
-    Unsigned 16-bit value
-
-Int32:
-    Signed 32-bit value
-
-UInt32:
-    Unsigned 32-bit value
-
-Int64:
-    Signed 64-bit value
-
-char[]:
-    Variable length single-byte character array (ANSI or UTF-8).
-
-FileTime:
-    Windows FILETIME structure.  64-bit value representing the number of
-100-nanosecond intervals since January 1, 1601 UTC.  Stored in local time.
-
-ShortString:
-    Variable length single-byte character string (ANSI or UTF-8).   Not null
-terminated.
-
-    Length   : UByte
-    Data     : char[Length]
-
-FileExString:
-    Variable length single-byte character string (UTF-8).  See "File Extended
-Header" section for details.
-
-
 ===============
 Content Records
 ===============
@@ -201,7 +207,8 @@ File Extended Headers
 =====================
 
 Like extended headers, file extended headers should be written in ascending
-numeric order.
+numeric order.  Multiple headers may occur within a single ID_FILE_EX record,
+and compliant parsers should break once they read a type they don't recognize.
 
 FILE_EX_VERSION (0x01)
     String representation of an executable file's Major/Minor/Maint/Build

data/bin/bc3_merge.rb

@@ -94,7 +94,7 @@ opts.on("-s", "--source  SOURCE", "Set source path for the snapshot") { |v|
 =begin
 Set target name.
 =end
-opts.on("-t", "--target  TARGET", "Set target file name") { |v|
+opts.on("-t", "--target  TARGET", "Set target file name (must be .yaml or .bcss)") { |v|
   $options[:target] = v
 }
 
@@ -113,6 +113,8 @@ opts.on("-i", "--initial SNAPSHOT_MASK", "Add snapshots without source-informati
   $options[:initial] << v
 }
 
+
+$log.info("Start bc_merge #{BC3::VERSION}")
 #Parsen der Parameter mit Exception bei ung�ltigen Parametern
 begin
 	opts.parse!
@@ -122,6 +124,7 @@ rescue OptionParser::MissingArgument, OptionParser::InvalidOption => err
 	puts "Error:\t#{err}"
 	#Ausgabe der Schnittstelle
 	puts opts
+  exit
 end
 
 snapshot = BC3::Snapshot.new( $options[:source] )
@@ -131,7 +134,7 @@ $options[:root].each{|mask|
   Dir[mask].each{|file|
     snap = BC3::SnapshotParser.new(file)
     snapshot << folder = BC3::Folder.new( snap.snapshot.path, snap.timestamp)
-    snap.snapshot.each{|key, x| folder << x }
+    snap.snapshot.each(:folders, :files, :rootonly ){|key, x| folder << x }
   }
 }
 
@@ -140,14 +143,14 @@ $options[:root].each{|mask|
   Dir[mask].each{|file|
     snap = BC3::SnapshotParser.new(file)
     snapshot << folder = BC3::Folder.new( File.basename(snap.snapshot.path), snap.timestamp )
-    snap.snapshot.each{|key, x| folder << x }
+    snap.snapshot.each(:folders, :files, :rootonly  ){|key, x| folder << x }
   }
 }
 
 $options[:initial].each{|mask|
   puts "Add #{mask} in raw-mode"
   Dir[mask].each{|file|
-    BC3::SnapshotParser.new(file).snapshot.each{|key, x| snapshot << x }
+    BC3::SnapshotParser.new(file).snapshot.each(:folders, :files, :rootonly ){|key, x| snapshot << x }
   }
 }
 

data/bin/bc3_search.rb

@@ -0,0 +1,342 @@
+#!/usr/bin/env ruby
+=begin rdoc
+Search inside bc3-snapshots.
+
+Please remember:
+BC3 offers itself a search function.
+
+This bc3_merge may be usefull, if you want to search in 
+many snapshots for specific files or directories.
+
+Usage:
+    bc3_search.rb 'pattern' *.bcss [options]
+    
+The ' around $pattern are needed, else the shell may expand the pattern as shell glob.
+
+Search is only possible on basenames (no pathes).
+
+==Pattern
+The search pattern handles the following parameters:
+* * Matches any character
+* ? Matches exactly one character
+* [] and {} may make problems.
+
+The pattern is translated to a regular expression.
+With option -r the pattern is used as a regular expression,
+no conversion is made.
+
+==Report the Result 
+===YAML-File
+Use option -y
+
+===BC3-Snapshot
+Use option -b (or --bcss) and the search result is
+saved as a snapshot.
+  bc_search -b extract *.bcss 
+Creates snapshots with the search result.
+The Filename is the original name with an additional 'extract'".
+
+  bc_search *.bcss -b
+Filename will get a "_extract_YYYY-MM-DD-hhmm".
+The option -b must be at the end (else the *.bcss will be taken to build a new filename)
+
+
+Use option -B (or --BCSS) and the global search result is
+saved as a snapshot.
+  bc_search -B extract.bcss *.bcss 
+Creates a snapshot 'extract.bcss' with all collected search results.
+
+  bc_search *.bcss -B
+Filename will get a "bc3_search_YYYY-MM-DD-hhmm.bcss".
+The option -B must be at the end (else the *.bcss will be taken to build a new filename)
+
+==Duplicates
+With option -d (or --duplicates) you can restrict the 
+results on duplicates inside a snapshot.
+
+With option -D (or --Duplicates) you can restrict the 
+results on duplicates in the snapshots.
+Only files with a duplicate in another (or the same) snapshot 
+are listed.
+
+The duplicate search can be combined with yaml or bcss-option.
+
+===BC3 method
+Beyond Compare offers itself a way to find duplicates:
+<em>
+The only way to find duplicate files is to flatten your folder structure so that all the files appear to be at the same level. You then can enable/show the CRC column, sort by it, then manually scan the file list for files with the same size and CRC code. I wouldn't advise it for a large number of files and folders, though. It could take a while to calculate all the CRC values, and it would be a tedious process to manually look through them all for duplicates.
+</em>
+http://www.scootersoftware.com/vbulletin/showpost.php?p=27736&postcount=4
+
+
+==Examples:
+  bc3_search.rb '*.rbw' *.bcss
+Search for rbw-files in all snapshots.
+
+===duplicate option
+  bc3_search.rb -d '*' *.bcss
+  bc3_search.rb --duplicates '*' *.bcss
+Search for duplicates inside all snapshots.
+Use the pattern '*' to check all files.
+
+  bc3_search.rb -d '*.rb' *.bcss
+Search for duplicate ruby scripts in all snapshots.
+
+
+  bc3_search.rb -D '*' *.bcss
+  bc3_search.rb --Duplicates '*' *.bcss
+Search for duplicates inside snapshots.
+Each snapshot lists all files, where you can find a duplicate
+in the same or another selected snapshot.
+
+
+===regexp option
+  bc3_search.rb -r '[ab].*\\.rbw' *.bcss
+  bc3_search.rb --regexp '[ab].*\\.rbw' *.bcss
+Search for rbw-files starting with an a or a b.
+The option -r switch to regular expression.
+
+===yaml option
+  bc3_search.rb '*.rbw' *.bcss -y
+  bc3_search.rb '*.rbw' *.bcss -yaml
+Search for rbw-files in all snapshots and 
+return the result in a yaml-list.
+
+=end
+
+=begin
+fixme/test:
+duplicates
+save as snapshot.
+=end
+
+$testmode = false
+#~ $testmode = true
+if $testmode
+puts "!!Testmode"
+#~ ARGV = %w{* ../unittest/test_search.bcss}
+#~ ARGV = %w{-r .* ../unittest/test_search.bcss}
+#~ ARGV = %w{-y * ../unittest/test_search.bcss}
+#~ ARGV = %w{-d * ../unittest/test_search.bcss}
+#~ ARGV = %w{-B * ../unittest/test_search.bcss}
+#~ ARGV = %w{-d * ../unittest/test_search.bcss -b}
+$:.unshift('../lib')
+end
+
+require 'optparse'
+require 'bc3'
+#~ $log.level = Log4r::DEBUG
+$log.level = Log4r::WARN
+
+
+#Collector
+$options = {
+  duplicates: false,
+  yaml: false,
+}
+
+=begin
+Create a Parser for command line
+=end
+opts = OptionParser.new()
+opts.banner = "Usage: bc3_search.rb pattern snapshots [options]"	#Usage-zeile
+opts.separator("Search inside Beyond Compare Snapshots")	
+
+=begin
+Pattern is a regular expression.
+=end
+opts.on('-r', "--[no-]regexp", "Use a ruby-regular expression") { |v|
+  $log.debug("Set usage of regular expression")
+  $options[:regexp] = v
+}
+
+=begin
+Define search for duplicates
+=end
+opts.on("-d", "--[no-]duplicates", "Restrict on duplicates inside snapshot") { |v|
+  $options[:duplicates] = v
+}
+opts.on("-D", "--Duplicates", "Search duplicates in all snapshots") { |v|
+  $options[:Duplicates] = true
+}
+
+
+=begin
+Puts result in yaml-style
+=end
+opts.on("-y", "--[no-]yaml", "Give result in yaml-style") { |v|
+  $options[:yaml] = v
+}
+
+=begin
+Save result as bcss-file
+=end
+opts.on("-b", "--bcss [FILENAME_ADDITION]", "Save result as bcss-file") { |v|
+  $options[:bcss] = v ? v : "extract_#{Time.now.strftime('%Y-%m-%d_%H%M')}"
+}
+opts.on("-B", "--BCSS [FILENAME]", "Save result as bcss-file") { |v|
+  $options[:BCSS] = v ? v : "bc3_search_#{Time.now.strftime('%Y-%m-%d_%H%M')}.bcss"
+}
+
+
+
+$log.info("Start bc_search #{BC3::VERSION}")
+#Parsen der Parameter mit Exception bei ung�ltigen Parametern
+begin
+	opts.parse!
+# rescue OptionParser::MissingArgument => err
+# rescue OptionParser::InvalidOption => err
+rescue OptionParser::MissingArgument, OptionParser::InvalidOption => err
+	puts "Error:\t#{err}"
+	#Ausgabe der Schnittstelle
+	puts opts
+  exit
+end
+
+if ARGV.empty?
+  puts "No search pattern and no Snapshot found"
+  exit
+end
+
+#Build regexp from pattern
+pattern = ARGV.shift.dup  #ARGV-parameter are frozen -> dup
+if ! $options[:regexp]
+  pattern.gsub!(/\./, '\.')
+  pattern.gsub!(/\*/, '.*')
+  pattern.gsub!(/\?/, '.')
+end
+begin 
+searchpattern = Regexp.new(pattern)
+rescue RegexpError
+  puts "Argument Error - Searchpattern has error"
+  exit
+end
+
+puts "Start bc3_search for pattern <#{searchpattern.source}>"
+if ARGV.empty?
+  puts "No Snapshot found"
+end
+
+if $options[:BCSS] and File.exist?($options[:BCSS])
+  puts "Target Snapshot #{$options[:BCSS]} already exists"
+  exit
+end
+
+puts "Restrict on global duplicates" if $options[:Duplicates]
+
+=begin
+Loop on non-option arguments and create snapshots.
+
+With option -D (or --Duplicates) a glogal crc-overview is generated.
+=end
+$snapshots = {}
+crc_list = {}
+ARGV.each{|filename|
+  if ! File.exist?(filename)
+    puts "Snapshot #{filename} not found"
+    next
+  end
+  $snapshots[filename] = BC3::SnapshotParser.new(filename).snapshot
+  if $options[:Duplicates]
+    $snapshots[filename].each(:files){|path, file|
+      ( crc_list[file.crc] ||= [] ) << path
+    }
+  end
+}
+
+$global_duplicates = []
+crc_list.each{| crc, filelist| $global_duplicates.concat(filelist) if filelist.size > 1 }
+
+#Collector for global hits
+$bcss = BC3::Snapshot.new( Time.now.strftime('bc3_search_%Y-%m-%d_%H%M.bcss'))
+
+
+=begin
+Loop on snapshots
+=end
+$snapshots.each{|file, snapshot|
+  puts "Analyse #{file} (#{snapshot.path})"
+  count = Hash.new(0)
+  bcss = BC3::Snapshot.new( snapshot.path + '_extract' )
+  
+=begin
+Prepare duplicate search if needed.
+=end
+  if $options[:duplicates]
+    snapshot.build_index
+    crc_list = snapshot.statistic['duplicates_by_crc']
+    
+    duplicates = []
+    if crc_list != 0
+      crc_list.each{| crc, filelist| duplicates.concat(filelist) }
+    else
+      puts "No duplicates found"
+    end
+  end
+=begin
+Loop on snapshot content
+=end
+  folders = []
+  snapshot.each(:recursive, :files, :folders){|key, element|
+    next unless File.basename(key) =~ searchpattern
+    next if $options[:duplicates] and ! duplicates.include?(key)
+    next if $options[:Duplicates] and ! $global_duplicates.include?(key)
+    
+    #Copy folder, but without content (content is no hit, only the folder itself).
+    if element.kind_of?(BC3::Folder)
+      element = BC3::Folder.new( element.dirname, element.timestamp, element.attributes )
+      folders << key  #separated collection of folders.
+    end
+    #Add to result
+    if key =~ /\/./
+      $bcss.basefolder.add_with_path(File.dirname(key), element) 
+      bcss.basefolder.add_with_path(File.dirname(key), element) 
+    else
+      $bcss << element
+      bcss << element
+    end
+    
+  } #snapshot
+  
+  if $options[:yaml]
+    puts bcss.to_hash.to_yaml
+  else
+    folders.each{|key| 
+      puts "\t#{key}"
+    }
+    bcss.each{|key,element| #only files
+      puts "\t#{key}"
+    }
+    {
+      :file => bcss.statistic['files'],
+      :folder => folders.size,
+    }.each{|key, value|
+      next if value == 0
+      puts "Found %3i %s%s in %s" % [ value, key, value > 1 ? 's' : '', file ]
+    }    
+  end
+
+=begin
+Create extract as snapshot.
+=end
+  if $options[:bcss]
+    filename = File.basename(file, '.bcss') + "_#{$options[:bcss]}.bcss"
+    if File.exist?(filename)
+      puts "Target Snapshot #{filename} already exists"
+      next
+    end
+    
+    puts "Save result in #{filename}"
+    bcss.save(filename)
+  end
+
+  puts '' #space for next snapshot
+} #snapshots
+
+=begin
+Create extract as snapshot.
+=end
+if $options[:BCSS]
+  puts "Save result in #{$options[:BCSS]}"
+  $bcss.save($options[:BCSS])
+end