format_parser 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fbc5d4476d94ccbeb0f50c3bd0406cb57d1fc5fa640afbbf8057d874ecc128a3
-  data.tar.gz: fa4421fa509d0f0528beb4c4b19dc1a63cd66fbed29142ed33b9b2aeae9067c0
+  metadata.gz: ebde95ad53411a3d4ae4db175bab7b90d568476302df66842c83710f88ad2e15
+  data.tar.gz: cfb0e6a00c9ca1f8447e71e0800643327b05114be1b592d888052ba94ed91a98
 SHA512:
-  metadata.gz: 6c3abee783f5b2c35da44231bbec2bfabd1ff25cafe45716694751f865e4cae6c71286dcf2053770f925391f8b3e0cef72daa351c43de03d0ed8b0900e4eabad
-  data.tar.gz: ab9ef11e37f5d2610ded657483a688876402cde506af5f830dd2f60fe2a8d432c182227a5c514050e79cfb059906fca2d384b366a6e64969ad7f5b5de81b8f43
+  metadata.gz: 1c6509362046f64b49472b3323f154ffc509f4f6d308d32ebc889299e470cc46d07cbe7864a5b8021a4e59d922d530cedf1e83852402305dab50f8c297ac9d8a
+  data.tar.gz: f8143722bd3dbc9b0b431732fc717d15dcd70c73e08bb2b76025f6e919976f6a62bb244f7525445387169e1ef7cea8e048084818255ea413458de5981e8fa0af

data/lib/care.rb

@@ -7,9 +7,9 @@ class Care
   DEFAULT_PAGE_SIZE = 128 * 1024
 
   class IOWrapper
-    def initialize(io, cache = Cache.new(DEFAULT_PAGE_SIZE))
+    def initialize(io, page_size: DEFAULT_PAGE_SIZE)
+      @cache = Cache.new(page_size)
       @io = io
-      @cache = cache
       @pos = 0
     end
 
@@ -109,6 +109,21 @@ class Care
       @pages[page_i] ||= read_page(io, page_i)
     end
 
+    def inspect
+      # To avoid page _contents_ in the inspect outputs we need to implement our own inspect.
+
+      # Simulate the builtin object ID output https://stackoverflow.com/a/11765495/153886
+      oid_str = (object_id << 1).to_s(16).rjust(16, '0')
+
+      ivars = instance_variables
+      ivars.delete(:@pages)
+      ivars_str = ivars.map do |ivar|
+        "#{ivar}=#{instance_variable_get(ivar).inspect}"
+      end.join(' ')
+      synthetic_vars = 'num_hydrated_pages=%d' % @pages.length
+      '#<%s:%s %s %s>' % [self.class, oid_str, synthetic_vars, ivars_str]
+    end
+
     def read_page(io, page_i)
       io.seek(page_i * @page_size)
       read_result = io.read(@page_size)

data/lib/format_parser.rb

@@ -9,16 +9,13 @@ module FormatParser
   require_relative 'archive'
   require_relative 'io_utils'
   require_relative 'read_limiter'
+  require_relative 'read_limits_config'
   require_relative 'remote_io'
   require_relative 'io_constraint'
   require_relative 'care'
 
   PARSER_MUX = Mutex.new
-
-  MAX_BYTES_READ_PER_PARSER = 512 * 1024
-  MAX_READS_PER_PARSER = 64 * 1024
-  MAX_SEEKS_PER_PARSER = 64 * 1024
-  MAX_CACHE_PAGE_FAULTS_PER_PARSER = 4
+  MAX_BYTES_READ_PER_PARSER = 1024 * 1024 * 2
 
   def self.register_parser(callable_or_responding_to_new, formats:, natures:)
     parser_provided_formats = Array(formats)
@@ -54,12 +51,20 @@ module FormatParser
 
   # Return all by default
   def self.parse(io, natures: @parsers_per_nature.keys, formats: @parsers_per_format.keys, results: :first)
+    # We need to apply various limits so that parsers do not over-read, do not cause too many HTTP
+    # requests to be dispatched and so on. These should be _balanced_ with one another- for example,
+    # we cannot tell a parser that it is limited to reading 1024 bytes while at the same time
+    # limiting the size of the cache pages it may slurp in to less than that amount, since
+    # it can quickly become frustrating. The limits configurator computes these limits
+    # for us, in a fairly balanced way, based on one setting.
+    limit_config = FormatParser::ReadLimitsConfig.new(MAX_BYTES_READ_PER_PARSER)
+
     # Limit the number of cached _pages_ we may fetch. This allows us to limit the number
     # of page faults (page cache misses) a parser may incur
-    read_limiter_under_cache = FormatParser::ReadLimiter.new(io, max_reads: MAX_CACHE_PAGE_FAULTS_PER_PARSER)
+    read_limiter_under_cache = FormatParser::ReadLimiter.new(io, max_reads: limit_config.max_pagefaults_per_parser)
 
     # Then configure a layer of caching on top of that
-    cached_io = Care::IOWrapper.new(read_limiter_under_cache)
+    cached_io = Care::IOWrapper.new(read_limiter_under_cache, page_size: limit_config.cache_page_size)
 
     # How many results has the user asked for? Used to determinate whether an array
     # is returned or not.
@@ -77,16 +82,16 @@ module FormatParser
     # between invocations, and would complicate threading situations
     parsers = parsers_for(natures, formats)
 
-    results = parsers.lazy.map do |parser|
-      # We need to rewind for each parser, anew
-      cached_io.seek(0)
+    # Limit how many operations the parser can perform
+    limited_io = ReadLimiter.new(cached_io, max_bytes: limit_config.max_read_bytes_per_parser, max_reads: limit_config.max_reads_per_parser, max_seeks: limit_config.max_seeks_per_parser)
 
-      # ...and we have to reset the cache page fault limit, each parser is allowed to cause N page faults
-      # - i.e. this is not a shared limit
+    results = parsers.lazy.map do |parser|
+      # Reset all the read limits, per parser
+      limited_io.reset_limits!
       read_limiter_under_cache.reset_limits!
 
-      # Limit how many operations the parser can perform
-      limited_io = ReadLimiter.new(cached_io, max_bytes: MAX_BYTES_READ_PER_PARSER, max_reads: MAX_READS_PER_PARSER, max_seeks: MAX_SEEKS_PER_PARSER)
+      # We need to rewind for each parser, anew
+      limited_io.seek(0)
 
       begin
         parser.call(limited_io)
@@ -107,6 +112,7 @@ module FormatParser
     end.reject(&:nil?).take(amount)
 
     return results.first if amount == 1
+
     # Convert the results from a lazy enumerator to an Array.
     results.to_a
   end

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.6.0'
+  VERSION = '0.7.0'
 end

data/lib/read_limits_config.rb

@@ -0,0 +1,29 @@
+class FormatParser::ReadLimitsConfig
+  MAX_PAGE_FAULTS = 8
+
+  def initialize(total_bytes_available_per_parser)
+    @max_read_bytes_per_parser = total_bytes_available_per_parser.to_i
+  end
+
+  def max_read_bytes_per_parser
+    @max_read_bytes_per_parser
+  end
+
+  def cache_page_size
+    @max_read_bytes_per_parser / 4
+  end
+
+  def max_pagefaults_per_parser
+    MAX_PAGE_FAULTS
+  end
+
+  def max_reads_per_parser
+    # Imagine we read per single byte
+    @max_read_bytes_per_parser / 2
+  end
+
+  def max_seeks_per_parser
+    # Imagine we have to seek once per byte
+    @max_read_bytes_per_parser / 2
+  end
+end

data/spec/care_spec.rb

@@ -4,6 +4,11 @@ describe Care do
   describe Care::Cache do
     let(:source) { StringIO.new('Hello there, this is our little caching reader') }
 
+    it 'provides #inspect but does not output the actual @pages instance variable with it' do
+      cache = Care::Cache.new(3)
+      expect(cache.inspect).not_to include('@pages')
+    end
+
     it 'performs correct reads at various offsets' do
       cache = Care::Cache.new(3)
       expect(cache.byteslice(source, 0, 3)).to eq('Hel')
@@ -72,9 +77,10 @@ describe Care do
       end
 
       cache_double = fake_cache_class.new
+      expect(Care::Cache).to receive(:new).and_return(cache_double)
       io_double = double('IO')
 
-      subject = Care::IOWrapper.new(io_double, cache_double)
+      subject = Care::IOWrapper.new(io_double)
 
       expect(subject.pos).to eq(0)
       subject.read(2)

data/spec/format_parser_spec.rb

@@ -31,9 +31,9 @@ describe FormatParser do
     }
     FormatParser.register_parser exploit, natures: :document, formats: :exploit
 
-    sample_io = StringIO.new(Random.new.bytes(1024 * 1024 * 4))
+    sample_io = StringIO.new(Random.new.bytes(1024 * 1024 * 8))
 
-    expect(sample_io).to receive(:read).at_most(4).times.and_call_original
+    expect(sample_io).to receive(:read).at_most(8).times.and_call_original
 
     result = FormatParser.parse(sample_io, formats: [:exploit])
     expect(result).to be_nil

data/spec/read_limits_config_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe FormatParser::ReadLimitsConfig do
+  it 'provides balanced values based on the initial byte read limit per parser' do
+    config = FormatParser::ReadLimitsConfig.new(1024)
+
+    expect(config.max_read_bytes_per_parser).to be_kind_of(Integer)
+    expect(config.max_read_bytes_per_parser).to be > 0
+
+    expect(config.cache_page_size).to be_kind_of(Integer)
+    expect(config.cache_page_size).to be > 0
+
+    expect(config.max_pagefaults_per_parser).to be_kind_of(Integer)
+    expect(config.max_pagefaults_per_parser).to be > 0
+
+    expect(config.max_reads_per_parser).to be_kind_of(Integer)
+    expect(config.max_reads_per_parser).to be > 0
+
+    expect(config.max_seeks_per_parser).to be_kind_of(Integer)
+    expect(config.max_seeks_per_parser).to be > 0
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Noah Berman
@@ -191,6 +191,7 @@ files:
 - lib/parsers/zip_parser/file_reader.rb
 - lib/parsers/zip_parser/office_formats.rb
 - lib/read_limiter.rb
+- lib/read_limits_config.rb
 - lib/remote_io.rb
 - lib/video.rb
 - spec/attributes_json_spec.rb
@@ -216,6 +217,7 @@ files:
 - spec/parsers/wav_parser_spec.rb
 - spec/parsers/zip_parser_spec.rb
 - spec/read_limiter_spec.rb
+- spec/read_limits_config_spec.rb
 - spec/remote_fetching_spec.rb
 - spec/remote_io_spec.rb
 - spec/spec_helper.rb