packfile_reader 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3b8e93c293d659c69561f903785550ebdcf9cbb4eff620d5f711a2eb55e2452f
-  data.tar.gz: 504bec783784ff1bd3233d7e39b1e53e3621239c79e54d6c54375358d69d504c
+  metadata.gz: ea469cdf74d5f5c3d95fc87de089c9ffe622f159200003d423e5c8d00e273ad5
+  data.tar.gz: aeea839368939c2b328c737c67acf987a94f67c1e3bfb5a6bd5a84c8afcf2d97
 SHA512:
-  metadata.gz: 06f757f92ed4ea72c888784ce94ede9d4502da4ee840f986ee74fb4fc984387387eba316ed7fc54dbb604d7df295f38bf5df589c0fee5bb962b49eac9fb5ebf7
-  data.tar.gz: 23dbeb6424004becebd49eb8823d14230a95fcd14e842054d3be46b54acfbee887e1d47d97f4f6f66e01d63b435d97331fe21daa27b6267f142230e5bdd14fad
+  metadata.gz: e5b3980bd08e06c19e8ca9ab9d46104b1a8282cf2e54129c3f63ee204eb0d477ea8b43316c361a70bba3a788c4d2fbfc7c66ea1812aa32581255f181ac0e7492
+  data.tar.gz: 95e3c21fa8c71e6bae77018fb18472055e0ceab34490103a683787e3f189615499eda5dadb850d28ad0c242400ca853f6acb1dfdaffb3445b3c705c17f395ade

data/README.md

@@ -39,7 +39,8 @@ where [options] are:
   -i, --filter-by-ids=<s>    Comma separated list of object ids to look for (default: any)
   -e, --expand-objects       Whether to expand objects data
   -o, --output-dir=<s>       Directory to store the expanded objects (default: .)
-  -v, --version              Print version and exit
+  -v, --verbose              Log some debugging informaiton to stderr
+  -r, --version              Print version and exit
   -l, --help                 Show this message
 
 ```
@@ -114,5 +115,18 @@ $ cat /tmp/5297f8f21ad868d9eb6a9c01ad09a9d186177047.txt
 # test-git%
 ```
 
+### Debugging information
+
+Passing `--verbose` to the command will add some debugging information to the output as well as the timestamp when the entry got processed at the beginning of the entry line.
+
+```
+packfile_reader --no-headers -verbose pack.sample 
+```
+
+```
+[2020-12-01 21:24:43 -0800] 96438dd1e26e6963fa65be0012e8f6e84209bc5d	OBJ_COMMIT	653
+[2020-12-01 21:24:43 -0800] 5297f8f21ad868d9eb6a9c01ad09a9d186177047	OBJ_BLOB	10
+[2020-12-01 21:24:44 -0800] bf195faf9d23ce0615cdefd2b746a077ef82f03f	OBJ_TREE	37
+```
 # References
 -  http://shafiul.github.io/gitbook/7_the_packfile.html

data/bin/packfile_reader

@@ -10,21 +10,22 @@ opts = Optimist::options do
     By default, the script will only report the object ids, their type and their deflated sizes.
     You can also make the script expand the content of the objects in the local directory or a directory
     of your choice.
- 
+
     Usage:
       packfile_reader [options] <packfile>
     where [options] are:
   EOS
- 
+
   opt :headers_only, 'Display only the headers of the packfile'
   opt :no_headers, 'Skip displaying the headers of the packfile'
   opt :filter_by_ids, 'Comma separated list of object ids to look for', :default => 'any', :short => '-i', :type => String
   opt :expand_objects, 'Whether to expand objects data', :default => false, :short => '-e'
   opt :output_dir, 'Directory to store the expanded objects', :default => '.', :short => '-o', :type => String
+  opt :verbose, 'Log some debugging informaiton to stderr', :default => false, :short => '-v'
 end
- 
+
 (puts "You must inform a single packfile, found #{ARGV.size}"; exit 1) if ARGV.size > 1 or ARGV.empty?
- 
+
 packfile = ARGV.first
 (puts 'Packfile not found'; exit 2) unless File.exist?(packfile)
 
@@ -50,13 +51,14 @@ File.open(packfile, 'rb') do |f|
   entries_processed = 0
   limit = objects_to_find == :any ? header.n_entries : objects_to_find.size
   (0...limit).each do
-    entry = PackfileReader::PackfileEntry.next_entry(f, objects_to_find) do |c,u,id|
+    entry = PackfileReader::PackfileEntry.next_entry(f, objects_to_find, opts[:verbose]) do |c,u,id|
       if opts[:expand_objects]
         dir = opts[:output_dir]
         File.open(File.join(dir, "#{id}.txt"), 'w') {|o| o.write u}
       end
     end
 
-    puts "#{entry.id}\t#{entry.type}\t#{entry.size}"
+    timestamp = opts[:verbose] ? "[#{Time.now}] " : ''
+    puts "\u001b[0K#{timestamp}#{entry.id}\t#{entry.type}\t#{entry.size}"
   end
-end
+end

data/lib/packfile_reader/packfile_entry.rb

@@ -17,7 +17,9 @@ module PackfileReader
 
     # Accepts a block that will receive the compressed data, uncompressed data and
     # the computed object id
-    def self.next_entry(packfile_io, objects_to_find=:any)
+    def self.next_entry(packfile_io, objects_to_find=:any, log_verbose=false)
+      raise 'Object id must be a valid sha1' unless objects_to_find == :any || objects_to_find.all? {|id| /^[0-9a-f]{40}$/.match? id }
+
       loop do
         return nil if packfile_io.eof?
 
@@ -27,15 +29,19 @@ module PackfileReader
         size = hunk.size
         offset = hunk.offset_size
 
+        # Clean the current line before printing the message
+        $stderr.puts "\u001b[0K>>>> Processing new entry [#{type}]" if log_verbose
         while hunk.continuation?
           hunk = PackfileReader::Hunk.new_without_type(packfile_io)
           size = (hunk.size << offset) | size # Data size is a combination of all hunk sizes
           offset += hunk.offset_size
         end
 
-        compressed_data, uncompressed_data = find_data(packfile_io)
+        compressed_data, uncompressed_data = find_data(packfile_io, log_verbose)
         object_id = compute_id(type, size, uncompressed_data)
 
+        type = "#{type} [CORRUPTED] " if uncompressed_data.nil?
+
         if objects_to_find == :any || objects_to_find.member?(object_id)
           yield compressed_data, uncompressed_data, object_id if block_given?
           return PackfileEntry.new(type, size, object_id)
@@ -44,7 +50,7 @@ module PackfileReader
     end
 
     private
-    def self.find_data(packfile_io)
+    def self.find_data(packfile_io, log_verbose)
       data_header = find_zlib_data_header(packfile_io)
 
       # since we don't have the index file that accompanies pack files
@@ -54,11 +60,16 @@ module PackfileReader
       compressed_data = data_header
       compressed_data += packfile_io.read(1)
 
+      bytes_read = 1
       begin
         uncompressed_data = Zlib.inflate(compressed_data)
       rescue Zlib::BufError
         compressed_data += packfile_io.read(1)
+        bytes_read += 1
+        $stderr.print " .... retrying on data gathering [#{bytes_read}] bytes read\r" if log_verbose
         retry
+      rescue Zlib::DataError
+        uncompressed_data = nil
       end
 
       [compressed_data, uncompressed_data]
@@ -92,7 +103,7 @@ module PackfileReader
       end
 
       return '000' if header_type.empty?
-      
+
       header = "#{header_type} #{size}\0"
       store = "#{header}#{uncompressed_data}"
       Digest::SHA1.hexdigest(store)
@@ -104,4 +115,4 @@ module PackfileReader
       @id = id
     end
   end
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: packfile_reader
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Robison WR Santos
@@ -56,7 +56,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Parses git packfiles without the help of idx companion