format_parser 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7a00aab34ab0fd5faeec5fa6069ab81b2d066b14
-  data.tar.gz: 51a4617a89c10524a88879cdabe34dd4ccdb24d3
+  metadata.gz: 99c08f5e2482e524cf7b6c54eb4de371bbf1a107
+  data.tar.gz: 3c33f9336c6ec7479b03102f8dbc4ab8610a6420
 SHA512:
-  metadata.gz: f9865f5245c785756a9ce84a89e2d17f2dbcf15a2172c3399d5112eb2d2f5ad613e233d641b31b47bce04d1d992a25a48d7e495b6025965e6599480c280dc9f6
-  data.tar.gz: 563cc7d7b3c8e6011ecc1d0dff1c245183c8d3da8a04c4669030ab009e6762183689f708f67b8b7ece4d7e2fe793f53771cf2efbcc1d186f473393bd4d76caac
+  metadata.gz: b9892a12a6007744c70baa01200d432b40868d6f8d854b506dea7349ef84282133f49ab244a9d383b7772221d29dc950ae07c604b57247784f42aae74a03474b
+  data.tar.gz: 643b274aa45ef8a11dde7e178d03036488767ba56f4e4680edf647ebb5098aff52c08bcecc9578c73bd6835aa235ac31358032806e0c28c5c113d8edad9734e1

data/format_parser.gemspec

@@ -39,5 +39,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'simplecov', '~> 0.15'
   spec.add_development_dependency 'pry', '~> 0.11'
   spec.add_development_dependency 'yard', '~> 0.9'
-  spec.add_development_dependency 'wetransfer_style', '0.4.0'
+  spec.add_development_dependency 'wetransfer_style', '0.5.0'
 end

data/lib/format_parser.rb

@@ -13,9 +13,11 @@ module FormatParser
   require_relative 'care'
 
   PARSER_MUX = Mutex.new
-  MAX_BYTES = 512 * 1024
-  MAX_READS = 64 * 1024
-  MAX_SEEKS = 64 * 1024
+
+  MAX_BYTES_READ_PER_PARSER = 512 * 1024
+  MAX_READS_PER_PARSER = 64 * 1024
+  MAX_SEEKS_PER_PARSER = 64 * 1024
+  MAX_CACHE_PAGE_FAULTS_PER_PARSER = 4
 
   def self.register_parser(callable_or_responding_to_new, formats:, natures:)
     parser_provided_formats = Array(formats)
@@ -46,24 +48,17 @@ module FormatParser
   end
 
   def self.parse_http(url, **kwargs)
-    remote_io = RemoteIO.new(url)
-    cached_io = Care::IOWrapper.new(remote_io)
-
-    # Prefetch the first page, since it is very likely to be touched
-    # by all parsers anyway. Additionally, when using RemoteIO we need
-    # to explicitly obtain the size of the resource, which is only available
-    # after having performed at least one successful GET - at least on S3
-    cached_io.read(1)
-    cached_io.seek(0)
-
-    parse(cached_io, **kwargs)
+    parse(RemoteIO.new(url), **kwargs)
   end
 
   # Return all by default
   def self.parse(io, natures: @parsers_per_nature.keys, formats: @parsers_per_format.keys, results: :first)
-    # If the cache is preconfigured do not apply an extra layer. It is going
-    # to be preconfigured when using parse_http.
-    io = Care::IOWrapper.new(io) unless io.is_a?(Care::IOWrapper)
+    # Limit the number of cached _pages_ we may fetch. This allows us to limit the number
+    # of page faults (page cache misses) a parser may incur
+    read_limiter_under_cache = FormatParser::ReadLimiter.new(io, max_reads: MAX_CACHE_PAGE_FAULTS_PER_PARSER)
+
+    # Then configure a layer of caching on top of that
+    cached_io = Care::IOWrapper.new(read_limiter_under_cache)
 
     # How many results has the user asked for? Used to determinate whether an array
     # is returned or not.
@@ -83,9 +78,15 @@ module FormatParser
 
     results = parsers.lazy.map do |parser|
       # We need to rewind for each parser, anew
-      io.seek(0)
+      cached_io.seek(0)
+
+      # ...and we have to reset the cache page fault limit, each parser is allowed to cause N page faults
+      # - i.e. this is not a shared limit
+      read_limiter_under_cache.reset_limits!
+
       # Limit how many operations the parser can perform
-      limited_io = ReadLimiter.new(io, max_bytes: MAX_BYTES, max_reads: MAX_READS, max_seeks: MAX_SEEKS)
+      limited_io = ReadLimiter.new(cached_io, max_bytes: MAX_BYTES_READ_PER_PARSER, max_reads: MAX_READS_PER_PARSER, max_seeks: MAX_SEEKS_PER_PARSER)
+
       begin
         parser.call(limited_io)
       rescue IOUtils::InvalidRead
@@ -95,6 +96,8 @@ module FormatParser
         # The parser tried to read too much - most likely the file structure
         # caused the parser to go off-track. Strictly speaking we should log this
         # and examine the file more closely.
+        # Or the parser caused too many cache pages to be fetched, which likely means we should not allow
+        # it to continue
       end
     end.reject(&:nil?).take(amount)
 

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.3.3'
+  VERSION = '0.3.4'
 end

data/lib/io_constraint.rb

@@ -1,4 +1,3 @@
-
 # We deliberately want to document and restrict the
 # number of methods an IO-ish object has to implement
 # to be usable with all our parsers. This subset is fairly

data/lib/parsers/exif_parser.rb

@@ -5,8 +5,8 @@ require 'delegate'
 class FormatParser::EXIFParser
   include FormatParser::IOUtils
 
-  # EXIFR kindly requests the presence of getbyte and readbyte
-  # IO methods, which our constrained IO subset does not provide natively
+  # EXIFR kindly requests the presence of a few more methods than what our IOConstraint
+  # is willing to provide, but they can be derived from the available ones
   class IOExt < SimpleDelegator
     def readbyte
       if byte = read(1)
@@ -14,6 +14,20 @@ class FormatParser::EXIFParser
       end
     end
 
+    def seek(n, seek_mode = IO::SEEK_SET)
+      io = __getobj__
+      case seek_mode
+      when IO::SEEK_SET
+        io.seek(n)
+      when IO::SEEK_CUR
+        io.seek(io.pos + n)
+      when IO::SEEK_END
+        io.seek(io.size + n)
+      else
+        raise Errno::EINVAL
+      end
+    end
+
     alias_method :getbyte, :readbyte
   end
 

data/lib/read_limiter.rb

@@ -45,4 +45,10 @@ class FormatParser::ReadLimiter
 
     @io.read(n)
   end
+
+  def reset_limits!
+    @seeks = 0
+    @reads = 0
+    @bytes = 0
+  end
 end

data/spec/format_parser_spec.rb

@@ -21,6 +21,26 @@ describe FormatParser do
     end
   end
 
+  it 'fails gracefully when a parser module reads more and more causing page faults and prevents too many reads on the source' do
+    exploit = ->(io) {
+      loop {
+        skip = 16 * 1024
+        io.read(1)
+        io.seek(io.pos + skip)
+      }
+    }
+    FormatParser.register_parser exploit, natures: :document, formats: :exploit
+
+    sample_io = StringIO.new(Random.new.bytes(1024 * 1024 * 4))
+
+    expect(sample_io).to receive(:read).at_most(4).times.and_call_original
+
+    result = FormatParser.parse(sample_io, formats: [:exploit])
+    expect(result).to be_nil
+
+    FormatParser.deregister_parser(exploit)
+  end
+
   describe 'multiple values return' do
     let(:blob) { StringIO.new(Random.new.bytes(512 * 1024)) }
     let(:audio) { FormatParser::Audio.new(format: :aiff, num_audio_channels: 1) }

data/spec/parsers/exif_parser_spec.rb

@@ -30,4 +30,30 @@ describe FormatParser::EXIFParser do
       end
     end
   end
+
+  describe 'IOExt' do
+    it 'supports readbyte' do
+      io = FormatParser::EXIFParser::IOExt.new(StringIO.new('hello'))
+      expect(io.readbyte).to eq(104)
+    end
+
+    it 'supports getbyte' do
+      io = FormatParser::EXIFParser::IOExt.new(StringIO.new('hello'))
+      expect(io.getbyte).to eq(104)
+    end
+
+    it 'supports seek modes' do
+      io = FormatParser::EXIFParser::IOExt.new(StringIO.new('hello'))
+      io.seek(1, IO::SEEK_SET)
+
+      io.seek(1, IO::SEEK_CUR)
+      expect(io.read(1)).to eq('l')
+
+      io.seek(-1, IO::SEEK_END)
+      expect(io.read(1)).to eq('o')
+
+      io.seek(1)
+      expect(io.read(1)).to eq('e')
+    end
+  end
 end

data/spec/read_limiter_spec.rb

@@ -41,4 +41,14 @@ describe FormatParser::ReadLimiter do
       reader.read(1)
     }.to raise_error(/bytes budget \(512\) exceeded/)
   end
+
+  it 'can be reset!' do
+    reader = FormatParser::ReadLimiter.new(io, max_bytes: 512)
+    reader.read(512)
+    expect {
+      reader.read(1)
+    }.to raise_error(/budget/)
+    reader.reset_limits!
+    reader.read(1)
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-02-27 00:00:00.000000000 Z
+date: 2018-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -129,14 +129,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 0.5.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 0.5.0
 description: |-
   A Ruby library for prying open files you can convert to a previewable format, such as video, image and audio files. It includes
     a number of parser modules that try to recover metadata useful for post-processing and layout while reading the absolute
@@ -230,7 +230,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: A library for efficient parsing of file metadata