git_ls 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ef6ee0fb0ab70df2190d65de7389011c84036fff2993ebf65d90913746f7ff3
-  data.tar.gz: 25c8eb1f6d58e830311b531f3aedb5f5302801bec42aded9e3df8d8e06f938a1
+  metadata.gz: 7736e59f5f16469e453f3f085e1473cef343eb79e51e75cb94c013de62a311ca
+  data.tar.gz: c9715b5c0080aea1d487b349d825f113d1ecf7d75a85fe76fb2cd9648f3c16c0
 SHA512:
-  metadata.gz: 31f293dba3526adcad121f36dd7afe09ca114a559581566e25ea6f60b071c566f29840aa2be46b61f6c3291f3da896b81f9b199e341ea535bb25695ae67381e8
-  data.tar.gz: 79401fdfd2f6ac2497f3554ceff5d27b9e690f31e2395dbf84a429f4ae8b2c257cd3c6a4cff35f3f5768f662566951edb1e28f59b8efe4abeb22f4ca9569b922
+  metadata.gz: ca243045d1d33fbc49532f35c63d8ea2bc924d119934a39407a9bc427e0e9de493997b6e93be6b3d5245c4d3df129b8bb1b696f0df6849bebdd4fc76febb718a
+  data.tar.gz: 8ebacdfaa8a7723fd87cf650ea72cf70140633b7140403bd0cbc0f320bc1bdfb28bfe8eeeb9fe950d697f2357b4352e5029ac9c749684bca097a2f05b02b4a1a

data/.rubocop.yml

@@ -118,6 +118,7 @@ Metrics/BlockLength:
     - describe
     - context
     - shared_examples
+    - benchmark
 
 Metrics/CyclomaticComplexity:
   Enabled: false
@@ -224,3 +225,20 @@ Style/HashTransformValues:
 
 Style/CommentedKeyword:
   Enabled: false
+
+Style/CaseLikeIf:
+  Enabled: false
+
+Style/NumericPredicate:
+  Enabled: false
+
+Style/Semicolon:
+  Exclude:
+    - bin/benchmark
+
+Naming/VariableNumber:
+  Enabled: false
+
+Lint/Void:
+  Exclude:
+    - bin/benchmark

data/.spellr_wordlists/english.txt

@@ -20,6 +20,7 @@ params
 pwd
 quotepath
 rdoc
+readme
 rspec
 rubo
 rubocop
@@ -30,4 +31,6 @@ simplecov
 sudo
 tmp
 usr
+utf
+warmup
 yardoc

data/.spellr_wordlists/ruby.txt

@@ -1,2 +1,4 @@
 ewah
+ips
 untr
+webpack

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# 0.5.0
+- Major Performance Improvements
+
 # 0.4.0
 - Handles split index files
 

data/Gemfile

@@ -4,6 +4,3 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in git_index.gemspec
 gemspec
-
-gem 'rake', '>= 12.3.3'
-gem 'rspec', '~> 3.0'

data/README.md

@@ -2,10 +2,9 @@
 
 Parses the .git/index file like `git ls-files` does.
 
-- for small repos (as in, anything smaller than rails),
-  it can be faster than doing the system call to git
-- still takes less than half a second for very large repos e.g. the linux repo
+- faster than doing the system call to git
 - doesn't require git to be installed
+- tested against ruby 2.4 - 2.7 and jruby
 
 ## Installation
 
@@ -23,23 +22,30 @@ Or install it yourself as:
 
     $ gem install git_ls
 
+And require
+```ruby
+require 'git_ls'
+```
+
 ## Usage
 
-`GitLS.files` returns an array of filenames, equivalent to `` `git ls-files -z`.split("\0") ``
+`GitLS.files` reads the `.git/index` file to return an array of file paths, equivalent to `` `git ls-files`.split("\n") ``, but faster, and without requiring git being installed.
+
 `GitLS.files("path/to/repo")` if the repo is not $PWD.
 
+Strictly speaking it's equivalent to `` `git ls-files -c core.quotepath=off -z`.split("\0") ``, handling file paths with spaces and non-ascii characters, and returning file paths as UTF-8 strings.
+
 ## Development
 
-Have a look in the bin dir for some useful tools.
-To install this gem onto your local machine, run `bundle exec rake install`.
+- Have a look in the bin dir for some useful tools.
+- To install this gem onto your local machine, run `bundle exec rake install`.
+- Run `rake` to run all tests & linters.
 
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/robotdana/git_ls.
 If you're comfortable, please attach `.git/index` (and `.git/sharedindex.<sha>` if applicable) and the output of `git ls-files` where it doesn't match.
 
-
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
-# git_ls

data/git_index.gemspec

@@ -27,10 +27,12 @@ Gem::Specification.new do |spec|
 
   spec.require_paths = ['lib']
 
+  spec.add_development_dependency 'benchmark-ips'
   spec.add_development_dependency 'bundler', '>= 2'
   spec.add_development_dependency 'leftovers'
   spec.add_development_dependency 'pry', '> 0'
   spec.add_development_dependency 'rake', '>= 12.3.3'
+  spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rubocop'
   spec.add_development_dependency 'rubocop-performance'
   spec.add_development_dependency 'rubocop-rspec'

data/lib/git_ls.rb

@@ -3,105 +3,100 @@
 # Usage:
 #   GitLS.files -> Array of strings as files.
 #   This will be identical output to git ls-files
+require 'stringio'
+
 module GitLS # rubocop:disable Metrics/ModuleLength
   class Error < StandardError; end
 
   class << self
-    def files(path = ::Dir.pwd)
-      read(path, false)
-    end
+    def files(path = nil)
+      path = path ? ::File.join(path, '.git/index') : '.git/index'
 
-    def headers(path = ::Dir.pwd)
-      read(path, true)
+      read(path, false)
     end
 
     private
 
-    def read(path, return_headers_only) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      path = ::File.join(path, '.git/index') if ::File.directory?(path)
-      file = ::File.new(path)
+    def read(path, _return_headers_only) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      begin
+        # reading the whole file into memory is faster than lots of ::File#read
+        # the biggest it's going to be is 10s of megabytes, well within ram.
+        file = ::StringIO.new(::File.read(path, mode: 'rb'))
+      rescue ::Errno::ENOENT => e
+        raise ::GitLS::Error, "Not a git directory: #{e.message}"
+      end
+
       buf = ::String.new
       # 4-byte signature:
       # The signature is { 'D', 'I', 'R', 'C' } (stands for "dircache")
       # 4-byte version number:
       # The current supported versions are 2, 3 and 4.
       # 32-bit number of index entries.
-      sig, git_index_version, length = file.read(12, buf).unpack('a4NN')
-      raise ::GitLS::Error, ".git/index file not found at #{path}" unless sig == 'DIRC'
+      sig = file.read(4, buf)
+      raise ::GitLS::Error, ".git/index file not found at '#{path}'" unless sig == 'DIRC'
 
-      return { git_index_version: git_index_version, length: length } if return_headers_only
+      git_index_version = file.read(4, buf).unpack1('N')
+      entries = file.read(4, buf).unpack1('N')
 
-      files = ::Array.new(length)
-      case git_index_version
+      files = ::Array.new(entries)
+      files = case git_index_version
       when 2 then files_2(files, file)
       when 3 then files_3(files, file)
       when 4 then files_4(files, file)
-      else raise ::GitLS::Error, 'Unrecognized git index version'
+      else raise ::GitLS::Error, "Unrecognized git index version '#{git_index_version}'"
       end
 
-      extensions(files, file, buf)
-      files
-    rescue ::Errno::ENOENT => e
-      raise ::GitLS::Error, "Not a git directory: #{e.message}"
-    ensure
-      # :nocov:
-      # coverage tracking for branches in ensure blocks is weird
-      file&.close
-      # :nocov:
-      files
+      read_extensions(files, file, path, buf)
     end
 
-    def extensions(files, file, buf)
-      case file.read(4, buf)
-      when 'link' then link_extension(files, file, buf)
-      when /[A-Z]{4}/ then ignored_extension(files, file, buf)
+    def read_extensions(files, file, path, buf) # rubocop:disable Metrics/MethodLength
+      extension = file.read(4, buf)
+      if extension == 'link'
+        read_link_extension(files, file, path, buf)
+      elsif extension.match?(/\A[A-Z]{4}\z/)
+        size = file.read(4, buf).unpack1('N')
+        file.seek(size, 1)
+        read_extensions(files, file, path, buf)
       else
-        return if (file.pos += 16) && file.eof?
+        return files if file.seek(16, 1) && file.eof?
 
-        raise ::GitLS::Error, "Unrecognized .git/index extension #{buf.inspect}"
+        raise ::GitLS::Error, "Unrecognized .git/index extension #{extension.inspect}"
       end
     end
 
-    def ignored_extension(files, file, buf)
-      size = file.read(4, buf).unpack1('N')
-      file.pos += size
-      extensions(files, file, buf)
-    end
-
-    def link_extension(files, file, buf) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      file.pos += 4 # size = file.read(4, buf).unpack1('N')
+    def read_link_extension(files, file, path, buf) # rubocop:disable Metrics/MethodLength
+      file.seek(4, 1) # skip size
 
       sha = file.read(20, buf)
 
-      new_files = files.dup
-
-      files.replace files("#{::File.dirname(file.path)}/sharedindex.#{sha.unpack1('H*')}")
+      split_files = read("#{::File.dirname(path)}/sharedindex.#{sha.unpack1('H*')}", false)
 
       ewah_each_value(file, buf) do |pos|
-        files[pos] = nil
+        split_files[pos] = nil
       end
 
       ewah_each_value(file, buf) do |pos|
-        replacement_file = new_files.shift
+        replacement_file = files.shift
         # the documentation *implies* that this *may* get a new filename
         # i can't get it to happen though
         # :nocov:
-        files[pos] = replacement_file unless replacement_file.empty?
+        split_files[pos] = replacement_file unless replacement_file.empty?
         # :nocov:
       end
 
-      files.compact!
-      files.concat(new_files)
-      files.sort!
+      split_files.compact!
+      split_files.concat(files)
+      split_files.sort!
 
-      extensions(files, file, buf)
+      read_extensions(split_files, file, path, buf)
     end
 
     # format is defined here:
     # https://git-scm.com/docs/bitmap-format#_appendix_a_serialization_format_for_an_ewah_bitmap
     def ewah_each_value(file, buf) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       uncompressed_pos = 0
-      file.pos += 4 # uncompressed_bits_count = file.read(4, buf).unpack1('N')
+
+      file.seek(4, 1) # skip 4 byte uncompressed_bits_count.
       compressed_bytes = file.read(4, buf).unpack1('N') * 8
 
       final_file_pos = file.pos + compressed_bytes
@@ -110,10 +105,10 @@ module GitLS # rubocop:disable Metrics/ModuleLength
         run_length_word = file.read(8, buf).unpack1('Q>')
         # 1st bit
         run_bit = run_length_word & 1
-        # the next 32 bits, masked, multiplied by 64 (which is shifted by 6 places)
-        run_length = ((run_length_word >> 1) & 0xFFFF_FFFF) << 6
+        # the next 32 bits, masked, multiplied by 64
+        run_length = ((run_length_word / 0b1_0) & 0xFFFF_FFFF) * 64
         # the next 31 bits
-        literal_length = (run_length_word >> 33)
+        literal_length = (run_length_word / 0b100000000_00000000_00000000_00000000_0)
 
         if run_bit == 1
           run_length.times do
@@ -124,9 +119,11 @@ module GitLS # rubocop:disable Metrics/ModuleLength
           uncompressed_pos += run_length
         end
 
-        literal_length.times do
-          word = file.read(8, buf).unpack1('B*').reverse
-          word.each_char do |char|
+        next unless literal_length > 0
+
+        words = file.read(8 * literal_length, buf).unpack('B64' * literal_length)
+        words.each do |word|
+          word.each_char.reverse_each do |char|
             yield(uncompressed_pos) if char == '1'
 
             uncompressed_pos += 1
@@ -134,47 +131,52 @@ module GitLS # rubocop:disable Metrics/ModuleLength
         end
       end
 
-      file.pos += 4 # bitmap metadata for adding to bitmaps
+      file.seek(4, 1) # bitmap metadata for adding to bitmaps
     end
 
     def files_2(files, file) # rubocop:disable Metrics/MethodLength
       files.map! do
-        file.pos += 60 # skip 60 bytes (40 bytes of stat, 20 bytes of sha)
-        length = ((file.getbyte & 0b0000_1111) << 8) + file.getbyte # find the 12 byte length
+        file.seek(60, 1) # skip 60 bytes (40 bytes of stat, 20 bytes of sha)
+
+        length = (file.getbyte & 0xF) * 256 + file.getbyte # find the 12 byte length
         if length < 0xFFF
           path = file.read(length)
           # :nocov:
         else
           # i can't test this i just get ENAMETOOLONG a lot
-          path = file.readline("\0").chop
-          file.pos -= 1
+          # I'm not sure it's even possible to get to this path, PATH_MAX is 4096 bytes on linux, 1024 on mac
+          # and length is a 12 byte number: 4096 max.
+          path = file.readline("\0").chop!
+          file.seek(-1, 1)
           # :nocov:
         end
-        file.pos += 8 - ((length - 2) % 8) # 1-8 bytes padding of nuls
+        file.seek(8 - ((length - 2) % 8), 1) # 1-8 bytes padding of nuls
+        path.force_encoding(Encoding::UTF_8)
         path
       end
     end
 
     def files_3(files, file) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       files.map! do
-        file.pos += 60 # skip 60 bytes (40 bytes of stat, 20 bytes of sha)
+        file.seek(60, 1) # skip 60 bytes (40 bytes of stat, 20 bytes of sha)
+        flags = file.getbyte
+        extended_flag = (flags & 0b0100_0000) > 0
+        length = (flags & 0xF) * 256 + file.getbyte # find the 12 byte length
+        file.seek(2, 1) if extended_flag
 
-        flags = file.getbyte * 256 + file.getbyte
-        extended_flag = (flags & 0b0100_0000_0000_0000).positive?
-        file.pos += 2 if extended_flag
-
-        length = flags & 0b0000_1111_1111_1111
         if length < 0xFFF
           path = file.read(length)
           # :nocov:
         else
           # i can't test this i just get ENAMETOOLONG a lot
-          path = file.readline("\0").chop
-          file.pos -= 1
+          # I'm not sure it's even possible to get to this path, PATH_MAX is 4096 bytes on linux, 1024 on mac
+          # and length is a 12 byte number: 4096 max.
+          path = file.readline("\0").chop!
+          file.seek(-1, 1)
           # :nocov:
         end
-
-        file.pos += 8 - ((path.bytesize - (extended_flag ? 0 : 2)) % 8) # 1-8 bytes padding of nuls
+        file.seek(8 - ((path.bytesize - (extended_flag ? 0 : 2)) % 8), 1) # 1-8 bytes padding of nuls
+        path.force_encoding(Encoding::UTF_8)
         path
       end
     end
@@ -182,11 +184,11 @@ module GitLS # rubocop:disable Metrics/ModuleLength
     def files_4(files, file) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       prev_entry_path = ''
       files.map! do
-        file.pos += 60 # skip 60 bytes (40 bytes of stat, 20 bytes of sha)
-        flags = file.getbyte * 256 + file.getbyte
-        file.pos += 2 if (flags & 0b0100_0000_0000_0000).positive?
-
-        length = flags & 0b0000_1111_1111_1111
+        file.seek(60, 1) # skip 60 bytes (40 bytes of stat, 20 bytes of sha)
+        flags = file.getbyte
+        extended_flag = (flags & 0b0100_0000) > 0
+        length = (flags & 0xF) * 256 + file.getbyte # find the 12 byte length
+        file.seek(2, 1) if extended_flag
 
         # documentation for this number from
         # https://git-scm.com/docs/pack-format#_original_version_1_pack_idx_files_have_the_following_format
@@ -199,8 +201,8 @@ module GitLS # rubocop:disable Metrics/ModuleLength
         read_offset = 0
         prev_read_offset = file.getbyte
         n = 1
-        while (prev_read_offset & 0b1000_0000).positive?
-          read_offset += (prev_read_offset - 0b1000_0000)
+        while (prev_read_offset & 0b1000_0000) > 0
+          read_offset += (prev_read_offset & 0b0111_1111)
           read_offset += 2**(7 * n)
           n += 1
           prev_read_offset = file.getbyte
@@ -211,15 +213,19 @@ module GitLS # rubocop:disable Metrics/ModuleLength
 
         if length < 0xFFF
           rest = file.read(length - initial_part_length)
-          file.pos += 1 # the NUL
+          file.seek(1, 1) # the NUL
           # :nocov:
         else
           # i can't test this i just get ENAMETOOLONG a lot
-          rest = file.readline("\0").chop
+          # I'm not sure it's even possible to get to this path, PATH_MAX is 4096 bytes on linux, 1024 on mac
+          # and length is a 12 byte number: 4096 max.
+          rest = file.readline("\0").chop!
+          file.seek(-1, 1)
           # :nocov:
         end
 
         prev_entry_path = prev_entry_path.byteslice(0, initial_part_length) + rest
+        prev_entry_path.force_encoding(Encoding::UTF_8)
       end
     end
   end

data/lib/git_ls/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GitLS
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: git_ls
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Dana Sherson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-05 00:00:00.000000000 Z
+date: 2020-09-12 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 12.3.3
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -183,7 +211,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Read a .git/index file and list the files