packfile_reader 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea469cdf74d5f5c3d95fc87de089c9ffe622f159200003d423e5c8d00e273ad5
-  data.tar.gz: aeea839368939c2b328c737c67acf987a94f67c1e3bfb5a6bd5a84c8afcf2d97
+  metadata.gz: 8eb4f8bc34379b5872964c517b718bbe1d5ee2488f02bc18983f3678be2b83da
+  data.tar.gz: 17effd8b76eca4b3bd7902b96b1a0c7494bdd977dbd11d9482ea31798cff5742
 SHA512:
-  metadata.gz: e5b3980bd08e06c19e8ca9ab9d46104b1a8282cf2e54129c3f63ee204eb0d477ea8b43316c361a70bba3a788c4d2fbfc7c66ea1812aa32581255f181ac0e7492
-  data.tar.gz: 95e3c21fa8c71e6bae77018fb18472055e0ceab34490103a683787e3f189615499eda5dadb850d28ad0c242400ca853f6acb1dfdaffb3445b3c705c17f395ade
+  metadata.gz: 66fda3747bc5d05ad5ae7ccd529831f5bf5491e4558fd1147bd562b4a7d9d157a9829d5fa5c5d9ae53dae36e66ecc8f4dec790f7eed7e35835cce44510fec0db
+  data.tar.gz: 7767f4c076cbfcafb99c4fd5219684eeba31b2081a9d86fcc4e9f8328159df86ee4950b497f8cb29148d7691803105a6e1e4e73f337df6800d9a1280b24073b7

data/README.md

@@ -6,7 +6,7 @@ Git packs up several of "loose" objects into a single binary file called a “pa
 
 The `.idx` file contains offsets for all the objects in the `.pack` file, so it is easier to find the content you are looking for on the packfile.
 
-When we have both files, we can use `git verify-pack' command to read the content and metadata about the objects in the packfile, but sometimes we only have the `.pack` file, and in this case `git` is not really helpful.
+When we have both files, we can use `git verify-pack` command to read the content and metadata about the objects in the packfile, but sometimes we only have the `.pack` file, and in this case `git` is not really helpful.
 
 ![packfile](packfile-format.png?raw=true "Packfile Format")
 
@@ -39,10 +39,10 @@ where [options] are:
   -i, --filter-by-ids=<s>    Comma separated list of object ids to look for (default: any)
   -e, --expand-objects       Whether to expand objects data
   -o, --output-dir=<s>       Directory to store the expanded objects (default: .)
+  -w, --window-size=<i>      Size of the window (in bytes) to use when looking for objects (default: 10000)
   -v, --verbose              Log some debugging informaiton to stderr
   -r, --version              Print version and exit
   -l, --help                 Show this message
-
 ```
 
 ## Example:
@@ -128,5 +128,14 @@ packfile_reader --no-headers -verbose pack.sample
 [2020-12-01 21:24:43 -0800] 5297f8f21ad868d9eb6a9c01ad09a9d186177047	OBJ_BLOB	10
 [2020-12-01 21:24:44 -0800] bf195faf9d23ce0615cdefd2b746a077ef82f03f	OBJ_TREE	37
 ```
+
+# Development
+
+## Testing
+
+```
+rake test
+```
+
 # References
 -  http://shafiul.github.io/gitbook/7_the_packfile.html

data/bin/packfile_reader

@@ -4,7 +4,7 @@ require 'optimist'
 require 'packfile_reader'
 
 opts = Optimist::options do
-  version "v0.0.1 (c) 2020 Robison WR Santos"
+  version "v0.0.2 (c) 2020 Robison WR Santos"
   banner <<~EOS
     This tool is used to parse and extract data from git packfiles without a .idx file.
     By default, the script will only report the object ids, their type and their deflated sizes.
@@ -21,6 +21,7 @@ opts = Optimist::options do
   opt :filter_by_ids, 'Comma separated list of object ids to look for', :default => 'any', :short => '-i', :type => String
   opt :expand_objects, 'Whether to expand objects data', :default => false, :short => '-e'
   opt :output_dir, 'Directory to store the expanded objects', :default => '.', :short => '-o', :type => String
+  opt :window_size, 'Size of the window (in bytes) to use when looking for objects', :default => 10_000, :short => '-w', :type => Integer
   opt :verbose, 'Log some debugging informaiton to stderr', :default => false, :short => '-v'
 end
 
@@ -51,7 +52,7 @@ File.open(packfile, 'rb') do |f|
   entries_processed = 0
   limit = objects_to_find == :any ? header.n_entries : objects_to_find.size
   (0...limit).each do
-    entry = PackfileReader::PackfileEntry.next_entry(f, objects_to_find, opts[:verbose]) do |c,u,id|
+    entry = PackfileReader::PackfileEntry.next_entry(f, objects_to_find, opts[:verbose], opts[:window_size]) do |c,u,id|
       if opts[:expand_objects]
         dir = opts[:output_dir]
         File.open(File.join(dir, "#{id}.txt"), 'w') {|o| o.write u}

data/lib/packfile_reader/packfile_entry.rb

@@ -16,8 +16,9 @@ module PackfileReader
     ]
 
     # Accepts a block that will receive the compressed data, uncompressed data and
-    # the computed object id
-    def self.next_entry(packfile_io, objects_to_find=:any, log_verbose=false)
+    # the computed object id. Window size is the amount of bytes to read at once
+    # while searching for the compressed data
+    def self.next_entry(packfile_io, objects_to_find=:any, log_verbose=false, window_size=10_000)
       raise 'Object id must be a valid sha1' unless objects_to_find == :any || objects_to_find.all? {|id| /^[0-9a-f]{40}$/.match? id }
 
       loop do
@@ -37,7 +38,7 @@ module PackfileReader
           offset += hunk.offset_size
         end
 
-        compressed_data, uncompressed_data = find_data(packfile_io, log_verbose)
+        compressed_data, uncompressed_data = find_data(packfile_io, log_verbose, window_size)
         object_id = compute_id(type, size, uncompressed_data)
 
         type = "#{type} [CORRUPTED] " if uncompressed_data.nil?
@@ -50,28 +51,35 @@ module PackfileReader
     end
 
     private
-    def self.find_data(packfile_io, log_verbose)
+    def self.find_data(packfile_io, log_verbose, window_size)
       data_header = find_zlib_data_header(packfile_io)
 
       # since we don't have the index file that accompanies pack files
       # we need to use brute force to find where the compressed data ends
-      # to do that, we go byte by byte and try to deflate the data, when
+      # to do that, we go <window_size> bytes by <window_size> bytes and try to deflate the data, when
       # that succeedes, we know we go it all
       compressed_data = data_header
-      compressed_data += packfile_io.read(1)
+      compressed_data += packfile_io.read(window_size)
 
-      bytes_read = 1
+      bytes_read = compressed_data.size
       begin
         uncompressed_data = Zlib.inflate(compressed_data)
       rescue Zlib::BufError
-        compressed_data += packfile_io.read(1)
-        bytes_read += 1
+        compressed_data += packfile_io.read(window_size)
+        bytes_read = compressed_data.size
         $stderr.print " .... retrying on data gathering [#{bytes_read}] bytes read\r" if log_verbose
         retry
       rescue Zlib::DataError
         uncompressed_data = nil
       end
 
+      $stderr.puts " .... data read [#{bytes_read}] bytes" if log_verbose
+      $stderr.print " .... repositioning file pointer to the end of current compressed data\r" if log_verbose
+      compressed_data = Zlib.deflate(uncompressed_data) if uncompressed_data
+
+      # reposition the file pointer to end of compressed data
+      packfile_io.seek(packfile_io.pos - (bytes_read - compressed_data.size))
+
       [compressed_data, uncompressed_data]
     end
 

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: packfile_reader
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Robison WR Santos
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
 date: 2020-11-30 00:00:00.000000000 Z
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.1
+        version: 3.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.1
+        version: 3.1.0
 description: A tool to parse git packfile when idx files are not present
 email: ''
 executables:
@@ -41,7 +41,7 @@ homepage: https://github.com/robisonsantos/packfile_reader
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -56,8 +56,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.33
+signing_key:
 specification_version: 4
 summary: Parses git packfiles without the help of idx companion
 test_files: []