packfile_reader 0.0.1 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3b8e93c293d659c69561f903785550ebdcf9cbb4eff620d5f711a2eb55e2452f
-  data.tar.gz: 504bec783784ff1bd3233d7e39b1e53e3621239c79e54d6c54375358d69d504c
+  metadata.gz: 8eb4f8bc34379b5872964c517b718bbe1d5ee2488f02bc18983f3678be2b83da
+  data.tar.gz: 17effd8b76eca4b3bd7902b96b1a0c7494bdd977dbd11d9482ea31798cff5742
 SHA512:
-  metadata.gz: 06f757f92ed4ea72c888784ce94ede9d4502da4ee840f986ee74fb4fc984387387eba316ed7fc54dbb604d7df295f38bf5df589c0fee5bb962b49eac9fb5ebf7
-  data.tar.gz: 23dbeb6424004becebd49eb8823d14230a95fcd14e842054d3be46b54acfbee887e1d47d97f4f6f66e01d63b435d97331fe21daa27b6267f142230e5bdd14fad
+  metadata.gz: 66fda3747bc5d05ad5ae7ccd529831f5bf5491e4558fd1147bd562b4a7d9d157a9829d5fa5c5d9ae53dae36e66ecc8f4dec790f7eed7e35835cce44510fec0db
+  data.tar.gz: 7767f4c076cbfcafb99c4fd5219684eeba31b2081a9d86fcc4e9f8328159df86ee4950b497f8cb29148d7691803105a6e1e4e73f337df6800d9a1280b24073b7

data/README.md

@@ -6,7 +6,7 @@ Git packs up several of "loose" objects into a single binary file called a “pa
 
 The `.idx` file contains offsets for all the objects in the `.pack` file, so it is easier to find the content you are looking for on the packfile.
 
-When we have both files, we can use `git verify-pack' command to read the content and metadata about the objects in the packfile, but sometimes we only have the `.pack` file, and in this case `git` is not really helpful.
+When we have both files, we can use `git verify-pack` command to read the content and metadata about the objects in the packfile, but sometimes we only have the `.pack` file, and in this case `git` is not really helpful.
 
 ![packfile](packfile-format.png?raw=true "Packfile Format")
 
@@ -39,9 +39,10 @@ where [options] are:
   -i, --filter-by-ids=<s>    Comma separated list of object ids to look for (default: any)
   -e, --expand-objects       Whether to expand objects data
   -o, --output-dir=<s>       Directory to store the expanded objects (default: .)
-  -v, --version              Print version and exit
+  -w, --window-size=<i>      Size of the window (in bytes) to use when looking for objects (default: 10000)
+  -v, --verbose              Log some debugging informaiton to stderr
+  -r, --version              Print version and exit
   -l, --help                 Show this message
-
 ```
 
 ## Example:
@@ -114,5 +115,27 @@ $ cat /tmp/5297f8f21ad868d9eb6a9c01ad09a9d186177047.txt
 # test-git%
 ```
 
+### Debugging information
+
+Passing `--verbose` to the command will add some debugging information to the output as well as the timestamp when the entry got processed at the beginning of the entry line.
+
+```
+packfile_reader --no-headers -verbose pack.sample 
+```
+
+```
+[2020-12-01 21:24:43 -0800] 96438dd1e26e6963fa65be0012e8f6e84209bc5d	OBJ_COMMIT	653
+[2020-12-01 21:24:43 -0800] 5297f8f21ad868d9eb6a9c01ad09a9d186177047	OBJ_BLOB	10
+[2020-12-01 21:24:44 -0800] bf195faf9d23ce0615cdefd2b746a077ef82f03f	OBJ_TREE	37
+```
+
+# Development
+
+## Testing
+
+```
+rake test
+```
+
 # References
 -  http://shafiul.github.io/gitbook/7_the_packfile.html

data/bin/packfile_reader

@@ -4,27 +4,29 @@ require 'optimist'
 require 'packfile_reader'
 
 opts = Optimist::options do
-  version "v0.0.1 (c) 2020 Robison WR Santos"
+  version "v0.0.2 (c) 2020 Robison WR Santos"
   banner <<~EOS
     This tool is used to parse and extract data from git packfiles without a .idx file.
     By default, the script will only report the object ids, their type and their deflated sizes.
     You can also make the script expand the content of the objects in the local directory or a directory
     of your choice.
- 
+
     Usage:
       packfile_reader [options] <packfile>
     where [options] are:
   EOS
- 
+
   opt :headers_only, 'Display only the headers of the packfile'
   opt :no_headers, 'Skip displaying the headers of the packfile'
   opt :filter_by_ids, 'Comma separated list of object ids to look for', :default => 'any', :short => '-i', :type => String
   opt :expand_objects, 'Whether to expand objects data', :default => false, :short => '-e'
   opt :output_dir, 'Directory to store the expanded objects', :default => '.', :short => '-o', :type => String
+  opt :window_size, 'Size of the window (in bytes) to use when looking for objects', :default => 10_000, :short => '-w', :type => Integer
+  opt :verbose, 'Log some debugging informaiton to stderr', :default => false, :short => '-v'
 end
- 
+
 (puts "You must inform a single packfile, found #{ARGV.size}"; exit 1) if ARGV.size > 1 or ARGV.empty?
- 
+
 packfile = ARGV.first
 (puts 'Packfile not found'; exit 2) unless File.exist?(packfile)
 
@@ -50,13 +52,14 @@ File.open(packfile, 'rb') do |f|
   entries_processed = 0
   limit = objects_to_find == :any ? header.n_entries : objects_to_find.size
   (0...limit).each do
-    entry = PackfileReader::PackfileEntry.next_entry(f, objects_to_find) do |c,u,id|
+    entry = PackfileReader::PackfileEntry.next_entry(f, objects_to_find, opts[:verbose], opts[:window_size]) do |c,u,id|
       if opts[:expand_objects]
         dir = opts[:output_dir]
         File.open(File.join(dir, "#{id}.txt"), 'w') {|o| o.write u}
       end
     end
 
-    puts "#{entry.id}\t#{entry.type}\t#{entry.size}"
+    timestamp = opts[:verbose] ? "[#{Time.now}] " : ''
+    puts "\u001b[0K#{timestamp}#{entry.id}\t#{entry.type}\t#{entry.size}"
   end
-end
+end

data/lib/packfile_reader/packfile_entry.rb

@@ -16,8 +16,11 @@ module PackfileReader
     ]
 
     # Accepts a block that will receive the compressed data, uncompressed data and
-    # the computed object id
-    def self.next_entry(packfile_io, objects_to_find=:any)
+    # the computed object id. Window size is the amount of bytes to read at once
+    # while searching for the compressed data
+    def self.next_entry(packfile_io, objects_to_find=:any, log_verbose=false, window_size=10_000)
+      raise 'Object id must be a valid sha1' unless objects_to_find == :any || objects_to_find.all? {|id| /^[0-9a-f]{40}$/.match? id }
+
       loop do
         return nil if packfile_io.eof?
 
@@ -27,15 +30,19 @@ module PackfileReader
         size = hunk.size
         offset = hunk.offset_size
 
+        # Clean the current line before printing the message
+        $stderr.puts "\u001b[0K>>>> Processing new entry [#{type}]" if log_verbose
         while hunk.continuation?
           hunk = PackfileReader::Hunk.new_without_type(packfile_io)
           size = (hunk.size << offset) | size # Data size is a combination of all hunk sizes
           offset += hunk.offset_size
         end
 
-        compressed_data, uncompressed_data = find_data(packfile_io)
+        compressed_data, uncompressed_data = find_data(packfile_io, log_verbose, window_size)
         object_id = compute_id(type, size, uncompressed_data)
 
+        type = "#{type} [CORRUPTED] " if uncompressed_data.nil?
+
         if objects_to_find == :any || objects_to_find.member?(object_id)
           yield compressed_data, uncompressed_data, object_id if block_given?
           return PackfileEntry.new(type, size, object_id)
@@ -44,23 +51,35 @@ module PackfileReader
     end
 
     private
-    def self.find_data(packfile_io)
+    def self.find_data(packfile_io, log_verbose, window_size)
       data_header = find_zlib_data_header(packfile_io)
 
       # since we don't have the index file that accompanies pack files
       # we need to use brute force to find where the compressed data ends
-      # to do that, we go byte by byte and try to deflate the data, when
+      # to do that, we go <window_size> bytes by <window_size> bytes and try to deflate the data, when
       # that succeedes, we know we go it all
       compressed_data = data_header
-      compressed_data += packfile_io.read(1)
+      compressed_data += packfile_io.read(window_size)
 
+      bytes_read = compressed_data.size
       begin
         uncompressed_data = Zlib.inflate(compressed_data)
       rescue Zlib::BufError
-        compressed_data += packfile_io.read(1)
+        compressed_data += packfile_io.read(window_size)
+        bytes_read = compressed_data.size
+        $stderr.print " .... retrying on data gathering [#{bytes_read}] bytes read\r" if log_verbose
         retry
+      rescue Zlib::DataError
+        uncompressed_data = nil
       end
 
+      $stderr.puts " .... data read [#{bytes_read}] bytes" if log_verbose
+      $stderr.print " .... repositioning file pointer to the end of current compressed data\r" if log_verbose
+      compressed_data = Zlib.deflate(uncompressed_data) if uncompressed_data
+
+      # reposition the file pointer to end of compressed data
+      packfile_io.seek(packfile_io.pos - (bytes_read - compressed_data.size))
+
       [compressed_data, uncompressed_data]
     end
 
@@ -92,7 +111,7 @@ module PackfileReader
       end
 
       return '000' if header_type.empty?
-      
+
       header = "#{header_type} #{size}\0"
       store = "#{header}#{uncompressed_data}"
       Digest::SHA1.hexdigest(store)
@@ -104,4 +123,4 @@ module PackfileReader
       @id = id
     end
   end
-end
+end

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: packfile_reader
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors:
 - Robison WR Santos
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
 date: 2020-11-30 00:00:00.000000000 Z
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.1
+        version: 3.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.1
+        version: 3.1.0
 description: A tool to parse git packfile when idx files are not present
 email: ''
 executables:
@@ -41,7 +41,7 @@ homepage: https://github.com/robisonsantos/packfile_reader
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -56,8 +56,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.33
+signing_key:
 specification_version: 4
 summary: Parses git packfiles without the help of idx companion
 test_files: []