format_parser 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ebde95ad53411a3d4ae4db175bab7b90d568476302df66842c83710f88ad2e15
-  data.tar.gz: cfb0e6a00c9ca1f8447e71e0800643327b05114be1b592d888052ba94ed91a98
+  metadata.gz: 789592107e9fa74091745b703249248b6a05e9dd73af45803ab799708f8498fc
+  data.tar.gz: 191ff20e5b8d455f681eb19b40a2bd406c1cfa1a9b2aeebaa483ea829071e532
 SHA512:
-  metadata.gz: 1c6509362046f64b49472b3323f154ffc509f4f6d308d32ebc889299e470cc46d07cbe7864a5b8021a4e59d922d530cedf1e83852402305dab50f8c297ac9d8a
-  data.tar.gz: f8143722bd3dbc9b0b431732fc717d15dcd70c73e08bb2b76025f6e919976f6a62bb244f7525445387169e1ef7cea8e048084818255ea413458de5981e8fa0af
+  metadata.gz: dde5379a4d0590019ac6eee21361e10688f8582efa169132f28a0dc531f6b05811368e987a3a30902dcc1a509952c11bb9810bee3f3dbd17e99f3c13f1776b71
+  data.tar.gz: ec5a699fb441ca622006fb2885a6e1ce623da0a7ea6a5b78e5df83db2a1f9227dd023681f12a7ec703188133a8276e363dd3e5c090bca3ce694064124925e47e

data/CHANGELOG.md

@@ -0,0 +1,98 @@
+## 0.8.0
+* Add `Measurometer` for applying instrumentation around FormatParser operaions. See documentation for usage.
+
+## 0.7.0
+* Configure read limits / pagefault limits centrally so that those limits make sense together
+
+## 0.6.0
+* Double the cache page size once more
+* We no longer need exifr/jpeg
+* Fix EXIF parsing in JPEG files
+* Reject Keynote documents in JPEG parser
+
+## 0.5.2
+* Do not raise EXIFR errors for keynote files
+* Correct broken comment for the audio nature
+
+## 0.5.1
+* Raise the cache page size during detection
+* Fix ZIP entry filename parsing
+
+## 0.5.0
+* Add FLAC parser
+* Add parse_atom_children_and_data_fields support
+* Add basic detection of Office files
+* Optimize EOCD signature lookup
+
+## 0.4.0
+* Adds a basic PDF parser
+* Make sure root: and to_json without arguments work
+* ZIP file format support
+
+## 0.3.5
+* Fix the bug with EXIF dimensions being used instead of pixel dimensions
+
+## 0.3.4
+* Pagefault limit
+* Add seek modes required by exifr
+
+## 0.3.3
+* Implement a sane to_json as well
+
+## 0.3.2
+* Add default as_json
+* Test on 2.5.0
+
+## 0.3.1
+* Remove post install warning
+* Moved aiff_parser_spec.rb to spec/parsers
+* CR2 file support
+* Add require 'set' to format_parser.rb
+* Use register_parser for natures/fmts
+
+## 0.3.0
+* Reverse API changes to support :first as default and add opts to parse_http
+* Implement and comply with rubocop
+* JPEG parser and Care fixes
+* Add format and count options to parse_http
+* Return first result as default
+* Use hashes for MOOV atom default fields
+
+## 0.2.0
+* Implement parser DSL
+
+## 0.1.7
+* Fix read(0) on Care::IOWrapper, introduce top-level tests
+
+## 0.1.6
+* Fix mp3 parsing bug
+* Add MOOV parser
+
+## 0.1.5
+* Add FDX parser
+* Remove dry-structs
+* New interface updates
+
+## 0.1.4
+* Add WAV parser
+
+## 0.1.3
+* Add MP3 parser
+* Add FileInformation#intrinsics
+* Disallow negative Care offsets
+
+## 0.1.2
+* Introduce a restrictive IO subset wrapper
+* Switch rewind for seek in exif parser
+* Prep for OSS release
+* Add fuzz spec
+* Improve orientation parsing
+* Optimisation for PNG and invalid input protection on JPEG
+
+## 0.1.1
+* Add AIFF parser
+
+## 0.1.0
+* Add parsers for PNG, JPG, TIFF, PSD
+* Add GIF parser
+* Add DPX parser

data/CONTRIBUTING.md

@@ -16,7 +16,7 @@ If you are interested in contributing code and would like to learn more about th
 
  - [ruby](https://ruby-doc.org)
  - [rspec](http://rspec.info/) (for testing)
- 
+
 # How do I make a contribution?
 
 ## Using the issue tracker
@@ -101,7 +101,7 @@ project's developers might not want to merge into the project.
 Please adhere to the coding conventions used throughout the project (indentation,
 accurate comments, etc.) and any other requirements (such as test coverage).
 
-The test suite can be run with `bundle exec rspec`. 
+The test suite can be run with `bundle exec rspec`.
 
 Follow this process if you'd like your work considered for inclusion in the
 project:
@@ -155,3 +155,7 @@ project:
 license your work under the same license as that used by the project, which you
 can see by clicking [here](https://github.com/WeTransfer/format_parser/blob/master/LICENSE.txt).
 This provision also applies to the test files you include with the changed code as fixtures.
+
+## Changelog
+
+When creating a new release you must add an entry in the `CHANGELOG.md`.

data/lib/care.rb

@@ -4,27 +4,49 @@
 # is only available via HTTP, for example, we can have less
 # fetches and have them return more data for one fetch
 class Care
+  # Defines the size of a page in bytes that the Care will prefetch
   DEFAULT_PAGE_SIZE = 128 * 1024
 
+  # Wraps any given IO with Care caching superpowers. Supports the subset
+  # of IO declared in IOConstraint.
   class IOWrapper
+    # Creates a new IOWrapper around the given source IO
+    #
+    # @param io[#seek, #pos, #size] the IO to wrap
+    # @param page_size[Integer] the size of the cache page to use for this wrapper
     def initialize(io, page_size: DEFAULT_PAGE_SIZE)
       @cache = Cache.new(page_size)
       @io = io
       @pos = 0
     end
 
+    # Returns the size of the resource contained in the IO
+    #
+    # @return Integer
     def size
       @io.size
     end
 
+    # Seeks the IO to the given absolute offset from the start of the file/resource
+    #
+    # @param to[Integer] offset in the IO
+    # @return Integer
     def seek(to)
       @pos = to
     end
 
+    # Returns the current position/offset within the IO
+    #
+    # @return Integer
     def pos
       @pos
     end
 
+    # Returns at most `n_bytes` of data from the IO or less if less data was available
+    # before the EOF was hit
+    #
+    # @param n_bytes[Integer]
+    # @return [String, nil] the content read from the IO or `nil` if no data was available
     def read(n_bytes)
       return '' if n_bytes == 0 # As hardcoded for all Ruby IO objects
       raise ArgumentError, "negative length #{n_bytes} given" if n_bytes < 0 # also as per Ruby IO objects
@@ -34,10 +56,17 @@ class Care
       read
     end
 
+    # Clears all the cached pages explicitly to help GC
+    #
+    # @return void
     def clear
       @cache.clear
     end
 
+    # Clears all the cached pages explicitly to help GC, and
+    # calls `#close` on the source IO if the IO responds to `#close`
+    #
+    # @return void
     def close
       clear
       @io.close if @io.respond_to?(:close)
@@ -47,6 +76,7 @@ class Care
   # Stores cached pages of data from the given IO as strings.
   # Pages are sized to be `page_size` or less (for the last page).
   class Cache
+    # Initializes a new cache pages container with pages of given size
     def initialize(page_size = DEFAULT_PAGE_SIZE)
       @page_size = page_size.to_i
       raise ArgumentError, 'The page size must be a positive Integer' unless @page_size > 0
@@ -59,6 +89,12 @@ class Care
     # If the IO has been exhausted, `nil` will be returned
     # instead. Will use the cached pages where available,
     # or fetch pages where necessary
+    #
+    # @param io[#seek, #read] the IO to read data from
+    # @param at[Integer] at which offset we have to read
+    # @param n_bytes[Integer] how many bytes we want to read/cache
+    # @return [String, nil] the content read from the IO or `nil` if no data was available
+    # @raise ArgumentError
     def byteslice(io, at, n_bytes)
       if n_bytes < 1
         raise ArgumentError, "The number of bytes to fetch must be a positive Integer, but was #{n_bytes}"
@@ -97,10 +133,18 @@ class Care
       slice if slice && !slice.empty?
     end
 
+    # Clears the page cache of all strings with data
+    #
+    # @return void
     def clear
       @pages.clear
     end
 
+    # Hydrates a page at the certain index or returns the contents of
+    # that page if it is already in the cache
+    #
+    # @param io[IO] the IO to read from
+    # @param page_i[Integer] which page (zero-based) to hydrate and return
     def hydrate_page(io, page_i)
       # Avoid trying to read the page if we know there is no content to fill it
       # in the underlying IO
@@ -109,9 +153,9 @@ class Care
       @pages[page_i] ||= read_page(io, page_i)
     end
 
+    # We provide an overridden implementation of #inspect to avoid
+    # printing the actual contents of the cached pages
     def inspect
-      # To avoid page _contents_ in the inspect outputs we need to implement our own inspect.
-
       # Simulate the builtin object ID output https://stackoverflow.com/a/11765495/153886
       oid_str = (object_id << 1).to_s(16).rjust(16, '0')
 
@@ -124,10 +168,15 @@ class Care
       '#<%s:%s %s %s>' % [self.class, oid_str, synthetic_vars, ivars_str]
     end
 
+    # Reads the requested page from the given IO
+    #
+    # @param io[IO] the IO to read from
+    # @param page_i[Integer] which page (zero-based) to read
     def read_page(io, page_i)
+      FormatParser::Measurometer.increment_counter('format_parser.parser.Care.page_reads_from_upsteam', 1)
+
       io.seek(page_i * @page_size)
       read_result = io.read(@page_size)
-
       if read_result.nil?
         # If the read went past the end of the IO the read result will be nil,
         # so we know our IO is exhausted here

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.7.0'
+  VERSION = '0.8.0'
 end

data/lib/format_parser.rb

@@ -1,5 +1,7 @@
 require 'set'
 
+# A pretty nimble module for parsing file metadata using partial reads. Contains all the
+# top-level methods of the library.
 module FormatParser
   require_relative 'attributes_json'
   require_relative 'image'
@@ -14,9 +16,19 @@ module FormatParser
   require_relative 'io_constraint'
   require_relative 'care'
 
+  # Is used to manage access to the shared array of parser constructors, which might
+  # potentially be mutated from different threads. The mutex won't be hit too often
+  # since it only locks when adding/removing parsers.
   PARSER_MUX = Mutex.new
   MAX_BYTES_READ_PER_PARSER = 1024 * 1024 * 2
 
+  # Register a parser object to be used to perform file format detection. Each parser FormatParser
+  # provides out of the box registers itself using this method.
+  #
+  # @param callable_or_responding_to_new[#call, #new] an object that either responds to #new or to #call
+  # @param formats[Array<Symbol>] file formats that the parser provides
+  # @param natures[Array<Symbol>] file natures that the parser provides
+  # @return void
   def self.register_parser(callable_or_responding_to_new, formats:, natures:)
     parser_provided_formats = Array(formats)
     parser_provided_natures = Array(natures)
@@ -36,6 +48,11 @@ module FormatParser
     end
   end
 
+  # Deregister a parser object (makes FormatParser forget this parser existed). Is mostly used in
+  # tests, but can also be used to forcibly disable some formats completely.
+  #
+  # @param callable_or_responding_to_new[#call, #new] an object that either responds to #new or to #call
+  # @return void
   def self.deregister_parser(callable_or_responding_to_new)
     # Used only in tests
     PARSER_MUX.synchronize do
@@ -45,11 +62,32 @@ module FormatParser
     end
   end
 
+  # Parses the resource at the given `url` and returns the results as if it were any IO
+  # given to `.parse`. The accepted keyword arguments are the same as the ones for `parse`.
+  #
+  # @param url[String, URI] the HTTP(S) URL to request the object from using Faraday and `Range:` requests
+  # @param kwargs the keyword arguments to be delegated to `.parse`
+  # @see {.parse}
   def self.parse_http(url, **kwargs)
     parse(RemoteIO.new(url), **kwargs)
   end
 
-  # Return all by default
+  # Parses the resource contained in the given IO-ish object, and returns either the first matched
+  # result (omitting all the other parsers), the first N results or all results.
+  #
+  # @param io[#seek, #pos, #read] an IO-ish object containing the resource to parse formats for
+  # @param natures[Array] an array of file natures to scope the parsing to.
+  #   For example `[:image]` will limit to image files.
+  #   The default value is "all natures known to FormatParser"
+  # @param formats[Array] an array of file formats to scope the parsing to.
+  #   For example `[:jpg, :tif]` will scope the parsing to TIFF and JPEG files.
+  #   The default value is "all formats known to FormatParser"
+  # @param results[:first, :all, Integer] one of the values defining how many results to return if parsing
+  #   is ambiguous. The default is `:first` which returns the first matching result. Other
+  #   possible values are `:all` to get all possible results and an Integer to return
+  #   at most N results.
+  # @return [Array<Result>, Result, nil] either an Array of results, a single parsing result or `nil`if
+  #   no useful metadata could be recovered from the file
   def self.parse(io, natures: @parsers_per_nature.keys, formats: @parsers_per_format.keys, results: :first)
     # We need to apply various limits so that parsers do not over-read, do not cause too many HTTP
     # requests to be dispatched and so on. These should be _balanced_ with one another- for example,
@@ -92,31 +130,59 @@ module FormatParser
 
       # We need to rewind for each parser, anew
       limited_io.seek(0)
-
-      begin
-        parser.call(limited_io)
-      rescue IOUtils::InvalidRead
-        # There was not enough data for this parser to work on,
-        # and it triggered an error
-      rescue IOUtils::MalformedFile
-        # Unexpected input was encountered during the parsing of
-        # a file. This might indicate either a malicious or a
-        # corruped file.
-      rescue ReadLimiter::BudgetExceeded
-        # The parser tried to read too much - most likely the file structure
-        # caused the parser to go off-track. Strictly speaking we should log this
-        # and examine the file more closely.
-        # Or the parser caused too many cache pages to be fetched, which likely means we should not allow
-        # it to continue
-      end
+      execute_parser_and_capture_expected_exceptions(parser, limited_io)
     end.reject(&:nil?).take(amount)
 
-    return results.first if amount == 1
-
     # Convert the results from a lazy enumerator to an Array.
-    results.to_a
+    results = results.to_a
+
+    if results.empty?
+      Measurometer.increment_counter('format_parser.unknown_files', 1)
+    end
+
+    amount == 1 ? results.first : results
+  end
+
+  def self.execute_parser_and_capture_expected_exceptions(parser, limited_io)
+    parser_name_for_instrumentation = parser.class.to_s.split('::').last
+    Measurometer.instrument('format_parser.parser.%s' % parser_name_for_instrumentation) do
+      parser.call(limited_io).tap do |result|
+        if result
+          Measurometer.increment_counter('format_parser.detected_natures.%s' % result.nature, 1)
+          Measurometer.increment_counter('format_parser.detected_formats.%s' % result.format, 1)
+        end
+      end
+    end
+  rescue IOUtils::InvalidRead
+    # There was not enough data for this parser to work on,
+    # and it triggered an error
+    Measurometer.increment_counter('format_parser.invalid_read_errors', 1)
+  rescue IOUtils::MalformedFile
+    # Unexpected input was encountered during the parsing of
+    # a file. This might indicate either a malicious or a
+    # corruped file.
+    Measurometer.increment_counter('format_parser.malformed_errors', 1)
+  rescue ReadLimiter::BudgetExceeded
+    # The parser tried to read too much - most likely the file structure
+    # caused the parser to go off-track. Strictly speaking we should log this
+    # and examine the file more closely.
+    # Or the parser caused too many cache pages to be fetched, which likely means we should not allow
+    # it to continue
+    Measurometer.increment_counter('format_parser.exceeded_budget_errors', 1)
+  ensure
+    limited_io.send_metrics(parser_name_for_instrumentation)
   end
 
+  # Returns objects that respond to `call` and can be called to perform parsing
+  # based on the _intersection_ of the two given nature/format constraints. For
+  # example, a constraint of "only image and only ZIP files" can be given -
+  # but would raise an error since no parsers provide both ZIP file parsing and
+  # images as their information.
+  #
+  # @param desired_natures[Array] which natures should be considered (like `[:image, :archive]`)
+  # @param desired_formats[Array] which formats should be considered (like `[:tif, :jpg]`)
+  # @return [Array<#call>] an array of callable parsers
+  # @raise ArgumentError when there are no parsers satisfying the constraint
   def self.parsers_for(desired_natures, desired_formats)
     assemble_parser_set = ->(hash_of_sets, keys_of_interest) {
       hash_of_sets.values_at(*keys_of_interest).compact.inject(&:+) || Set.new
@@ -133,6 +199,11 @@ module FormatParser
     factories.map { |callable_or_class| instantiate_parser(callable_or_class) }
   end
 
+  # Instantiates a parser object (an object that responds to `#call`) from a given class
+  # or returns the parameter as is if it is callable by itself - i.e. if it is a Proc
+  #
+  # @param callable_or_responding_to_new[#call, #new] a callable or a Class/Module
+  # @return [#call] a parser that can be called with an IO-ish argument
   def self.instantiate_parser(callable_or_responding_to_new)
     if callable_or_responding_to_new.respond_to?(:call)
       callable_or_responding_to_new
@@ -146,4 +217,7 @@ module FormatParser
   Dir.glob(__dir__ + '/parsers/*.rb').sort.each do |parser_file|
     require parser_file
   end
+  # The Measurometer latches itself onto existing classes, so load it after
+  # we have loaded all the parsers
+  require_relative 'measurometer'
 end

data/lib/io_constraint.rb

@@ -19,18 +19,33 @@ class FormatParser::IOConstraint
     @io = io
   end
 
+  # Returns at most `n_bytes` of data from the IO or less if less data was available
+  # before the EOF was hit
+  #
+  # @param n_bytes[Integer]
+  # @return [String, nil] the content read from the IO or `nil` if no data was available
   def read(n_bytes)
     @io.read(n_bytes)
   end
 
-  def seek(absolute_offset)
-    @io.seek(absolute_offset)
+  # Seeks the IO to the given absolute offset from the start of the file/resource
+  #
+  # @param to[Integer] offset in the IO
+  # @return Integer
+  def seek(to)
+    @io.seek(to)
   end
 
+  # Returns the size of the resource contained in the IO
+  #
+  # @return Integer
   def size
     @io.size
   end
 
+  # Returns the current position/offset within the IO
+  #
+  # @return Integer
   def pos
     @io.pos
   end

data/lib/io_utils.rb

@@ -26,11 +26,7 @@ module FormatParser::IOUtils
 
     raise InvalidRead, 'Negative skips are not supported' if n < 0
 
-    if io.respond_to?(:pos)
-      io.seek(io.pos + n)
-    else
-      safe_read(io, n)
-    end
+    io.seek(io.pos + n)
     nil
   end
 

data/lib/measurometer.rb

@@ -0,0 +1,100 @@
+class FormatParser::Measurometer
+  class << self
+    # Permits adding instrumentation drivers. Measurometer is 1-1 API
+    # compatible with Appsignal, which we use a lot. So to magically
+    # obtain all Appsignal instrumentation, add the Appsignal module
+    # as a driver.
+    #
+    #   Measurometer.drivers << Appsignal
+    #
+    # A driver must be reentrant and thread-safe - it should be possible
+    # to have multiple `instrument` calls open from different threads at the
+    # same time.
+    # The driver must support the same interface as the Measurometer class
+    # itself, minus the `drivers` and `instrument_instance_method` methods.
+    #
+    # @return Array
+    def drivers
+      @drivers ||= []
+      @drivers
+    end
+
+    # Runs a given block within a cascade of `instrument` blocks of all the
+    # added drivers.
+    #
+    #   Measurometer.instrument('do_foo') { compute! }
+    #
+    # unfolds to
+    #   Appsignal.instrument('do_foo') do
+    #     Statsd.timing('do_foo') do
+    #       compute!
+    #     end
+    #   end
+    #
+    # A driver must be reentrant and thread-safe - it should be possible
+    # to have multiple `instrument` calls open from different threads at the
+    # same time.
+    # The driver must support the same interface as the Measurometer class
+    # itself, minus the `drivers` and `instrument_instance_method` methods.
+    #
+    # @param block_name[String] under which path to push the metric
+    # @param blk[#call] the block to instrument
+    # @return [Object] the return value of &blk
+    def instrument(block_name, &blk)
+      return yield unless @drivers && @drivers.any? # The block wrapping business is not free
+      @drivers.inject(blk) { |outer_block, driver|
+        -> {
+          driver.instrument(block_name, &outer_block)
+        }
+      }.call
+    end
+
+    # Adds a distribution value (sample) under a given path
+    #
+    # @param value_path[String] under which path to push the metric
+    # @param value[Numeric] distribution value
+    # @return nil
+    def add_distribution_value(value_path, value)
+      (@drivers || []).each { |d| d.add_distribution_value(value_path, value) }
+      nil
+    end
+
+    # Increment a named counter under a given path
+    #
+    # @param counter_path[String] under which path to push the metric
+    # @param by[Integer] the counter increment to apply
+    # @return nil
+    def increment_counter(counter_path, by)
+      (@drivers || []).each { |d| d.increment_counter(counter_path, by) }
+      nil
+    end
+
+    # Wrap an anonymous module around an instance method in the given class to have
+    # it instrumented automatically. The name of the measurement will be interpolated as:
+    #
+    #   "#{prefix}.#{rightmost_class_constant_name}.#{instance_method_name}"
+    #
+    # @param target_class[Class] the class to instrument
+    # @param instance_method_name_to_instrument[Symbol] the method name to instrument
+    # @param path_prefix[String] under which path to push the instrumented metric
+    # @return void
+    def instrument_instance_method(target_class, instance_method_name_to_instrument, path_prefix)
+      short_class_name = target_class.to_s.split('::').last
+      instrumentation_name = [path_prefix, short_class_name, instance_method_name_to_instrument].join('.')
+      instrumenter_module = Module.new do
+        define_method(instance_method_name_to_instrument) do |*any|
+          ::FormatParser::Measurometer.instrument(instrumentation_name) { super(*any) }
+        end
+      end
+      target_class.prepend(instrumenter_module)
+    end
+  end
+
+  # Instrument things interesting in the global sense
+  instrument_instance_method(FormatParser::RemoteIO, :read, 'format_parser')
+  instrument_instance_method(Care::Cache, :read_page, 'format_parser')
+
+  # Instrument more specific things on a per-parser basis
+  instrument_instance_method(FormatParser::EXIFParser, :scan_image_tiff, 'format_parser')
+  instrument_instance_method(FormatParser::MOOVParser::Decoder, :extract_atom_stream, 'format_parser.parsers.MOOVParser')
+end

data/lib/parsers/jpeg_parser.rb

@@ -112,6 +112,7 @@ class FormatParser::JPEGParser
     maybe_exif_magic_str = app1_frame_bytes[0..5]
     maybe_exif_data = app1_frame_bytes[6..-1]
     if maybe_exif_magic_str == EXIF_MAGIC_STRING
+      FormatParser::Measurometer.add_distribution_value('format_parser.JPEGParser.bytes_sent_to_exif_parser', maybe_exif_data.bytesize)
       scanner = FormatParser::EXIFParser.new(StringIO.new(maybe_exif_data))
       scanner.scan_image_tiff
 

data/lib/parsers/moov_parser.rb

@@ -11,10 +11,6 @@ class FormatParser::MOOVParser
     'm4a ' => :m4a,
   }
 
-  # It is currently not documented and not particularly well-tested,
-  # so not considered a public API for now
-  private_constant :Decoder
-
   def call(io)
     return unless matches_moov_definition?(io)
 

data/lib/read_limiter.rb

@@ -1,9 +1,17 @@
+# Is used to limit the number of reads/seeks parsers can perform
 class FormatParser::ReadLimiter
   NO_LIMIT = nil
 
   class BudgetExceeded < StandardError
   end
 
+  # Creates a ReadLimiter wrapper around the given IO object and sets the limits
+  # on the number of reads/writes
+  #
+  # @param io[#seek, #pos, #size, #read] the IO object to wrap
+  # @param max_bytes[Integer, nil] how many bytes can we read from this before an exception is raised
+  # @param max_reads[Integer, nil] how many read() calls can we perform on this before an exception is raised
+  # @param max_seeks[Integer, nil] how many seek() calls can we perform on this before an exception is raised
   def initialize(io, max_bytes: NO_LIMIT, max_reads: NO_LIMIT, max_seeks: NO_LIMIT)
     @max_bytes = max_bytes
     @max_reads = max_reads
@@ -15,24 +23,39 @@ class FormatParser::ReadLimiter
     @bytes = 0
   end
 
+  # Returns the size of the resource contained in the IO
+  #
+  # @return Integer
   def size
     @io.size
   end
 
+  # Returns the current position/offset within the IO
+  #
+  # @return Integer
   def pos
     @io.pos
   end
 
-  def seek(to_offset)
+  # Seeks the IO to the given absolute offset from the start of the file/resource
+  #
+  # @param to[Integer] offset in the IO
+  # @return Integer
+  def seek(to)
     @seeks += 1
     if @max_seeks && @seeks > @max_seeks
       raise BudgetExceeded, 'Seek budget exceeded (%d seeks performed)' % @max_seeks
     end
-    @io.seek(to_offset)
+    @io.seek(to)
   end
 
-  def read(n)
-    @bytes += n
+  # Returns at most `n_bytes` of data from the IO or less if less data was available
+  # before the EOF was hit
+  #
+  # @param n_bytes[Integer]
+  # @return [String, nil] the content read from the IO or `nil` if no data was available
+  def read(n_bytes)
+    @bytes += n_bytes
     @reads += 1
 
     if @max_bytes && @bytes > @max_bytes
@@ -43,9 +66,23 @@ class FormatParser::ReadLimiter
       raise BudgetExceeded, 'Number of read() calls exceeded (%d max)' % @max_reads
     end
 
-    @io.read(n)
+    @io.read(n_bytes)
   end
 
+  # Sends the metrics about the state of this ReadLimiter to a Measurometer
+  #
+  # @param prefix[String] the prefix to set. For example, with prefix "TIFF" the metrics will be called
+  #   `format_parser.TIFF.read_limiter.num_seeks` and so forth
+  # @return void
+  def send_metrics(prefix)
+    FormatParser::Measurometer.add_distribution_value('format_parser.%s.read_limiter.num_seeks' % prefix, @seeks)
+    FormatParser::Measurometer.add_distribution_value('format_parser.%s.read_limiter.num_reads' % prefix, @reads)
+    FormatParser::Measurometer.add_distribution_value('format_parser.%s.read_limiter.read_bytes' % prefix, @bytes)
+  end
+
+  # Resets all the recorded call counters so that the object can be reused for the next parser,
+  # which will have it's own limits
+  # @return void
   def reset_limits!
     @seeks = 0
     @reads = 0

data/lib/read_limits_config.rb

@@ -5,23 +5,49 @@ class FormatParser::ReadLimitsConfig
     @max_read_bytes_per_parser = total_bytes_available_per_parser.to_i
   end
 
+  # Defines how many bytes each parser may request to read from the IO object given to it.
+  # Is used to artificially limit unbounded reads in parsers that may wander off and
+  # try to gulp in the file given to them indefinitely due to infinite loops or
+  # wrongly implemented skips - or when handling data that has been deliberately
+  # crafted in a way that can make a parser misbehave.
+  # This is less strict than one could think - for example, the MOOV parser used for
+  # Quicktime files will skip over the actual atom contents of the atoms, and will only
+  # read atom headers - which stays under this limit for quite some time.
   def max_read_bytes_per_parser
     @max_read_bytes_per_parser
   end
 
+  # How big should the cache page be. Each cache page read will incur one `#read`
+  # on the underlying IO object, remote or local
   def cache_page_size
     @max_read_bytes_per_parser / 4
   end
 
+  # Each parser can incur HTTP requests when performing `parse_http`. This constant
+  # sets the maximum number of pages each parser is allowed to hit that have not
+  # been fetched previously and are not stored in the cache. For example, with most
+  # formats the first cache page and the last cache page - tail and head of the file,
+  # respectively - will be available right after the first parser retreives some data.
+  # The second parser accessing the same data will reuse the in-memory cache.
   def max_pagefaults_per_parser
     MAX_PAGE_FAULTS
   end
 
+  # Defines how many `#read` calls each parser may perform on the IO object given to it.
+  # Is used to artificially limit unbounded reads in parsers that may wander off and
+  # try to gulp in the file given to them indefinitely due to infinite loops or
+  # wrongly implemented skips - or when handling data that has been deliberately
+  # crafted in a way that can make a parser misbehave.
   def max_reads_per_parser
     # Imagine we read per single byte
     @max_read_bytes_per_parser / 2
   end
 
+  # Defines how many `#seek` calls each parser may perform on the IO object given to it.
+  # Is used to artificially limit unbounded reads in parsers that may wander off and
+  # try to gulp in the file given to them indefinitely due to infinite loops or
+  # wrongly implemented skips - or when handling data that has been deliberately
+  # crafted in a way that can make a parser misbehave.
   def max_seeks_per_parser
     # Imagine we have to seek once per byte
     @max_read_bytes_per_parser / 2

data/lib/remote_io.rb

@@ -1,3 +1,8 @@
+# Acts as a wrapper for turning a given URL into an IO object
+# you can read from and seek in. Uses Faraday under the hood
+# to perform fetches, so if you apply Faraday configuration
+# tweaks using `Faraday.default_connection = ...` these will
+# take effect for these RemoteIO objects as well
 class FormatParser::RemoteIO
   # Represents a failure that might be retried
   # (like a 5xx response or a timeout)
@@ -89,8 +94,10 @@ class FormatParser::RemoteIO
       # cannot hint size with this response - at lease not when working with S3
       return
     when 500..599
+      FormatParser::Measurometer.increment_counter('format_parser.RemoteIO.upstream50x_errors', 1)
       raise IntermittentFailure, "Server at #{@uri} replied with a #{response.status} and we might want to retry"
     else
+      FormatParser::Measurometer.increment_counter('format_parser.RemoteIO.invalid_request_errors', 1)
       raise InvalidRequest, "Server at #{@uri} replied with a #{response.status} and refused our request"
     end
   end

data/spec/io_utils_spec.rb

@@ -27,16 +27,10 @@ describe 'IOUtils' do
       }.to raise_error(FormatParser::IOUtils::InvalidRead)
     end
 
-    it 'uses #pos if available on the object' do
+    it 'uses #pos available on the object' do
       fake_io = double(pos: 11)
       expect(fake_io).to receive(:seek).with(11 + 5)
       safe_skip(fake_io, 5)
     end
-
-    it 'uses #read if no #pos is available on the object' do
-      fake_io = double
-      expect(fake_io).to receive(:read).with(5).and_return('x' * 5)
-      safe_skip(fake_io, 5)
-    end
   end
 end

data/spec/measurometer_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+describe FormatParser::Measurometer do
+  RSpec::Matchers.define :include_counter_or_measurement_named do |named|
+    match do |actual|
+      actual.any? do |e|
+        e[0] == named && e[1] > 0
+      end
+    end
+  end
+
+  it 'instruments a full cycle FormatParser.parse' do
+    driver_class = Class.new do
+      attr_accessor :timings, :counters, :distributions
+      def instrument(block_name)
+        s = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        yield.tap do
+          delta = Process.clock_gettime(Process::CLOCK_MONOTONIC) - s
+          @timings ||= []
+          @timings << [block_name, delta * 1000]
+        end
+      end
+
+      def add_distribution_value(value_path, value)
+        @distributions ||= []
+        @distributions << [value_path, value]
+      end
+
+      def increment_counter(value_path, value)
+        @counters ||= []
+        @counters << [value_path, value]
+      end
+    end
+
+    instrumenter = driver_class.new
+    described_class.drivers << instrumenter
+
+    FormatParser.parse(File.open(fixtures_dir + 'JPEG/keynote_recognized_as_jpeg.key', 'rb'), results: :all)
+
+    described_class.drivers.delete(instrumenter)
+    expect(described_class.drivers).not_to include(instrumenter)
+
+    expect(instrumenter.counters).to include_counter_or_measurement_named('format_parser.detected_formats.zip')
+    expect(instrumenter.counters).to include_counter_or_measurement_named('format_parser.parser.Care.page_reads_from_upsteam')
+    expect(instrumenter.distributions).to include_counter_or_measurement_named('format_parser.ZIPParser.read_limiter.read_bytes')
+    expect(instrumenter.timings).to include_counter_or_measurement_named('format_parser.Cache.read_page')
+  end
+end

data/spec/remote_fetching_spec.rb

@@ -27,11 +27,12 @@ describe 'Fetching data from HTTP remotes' do
   end
 
   it '#parse_http is called with hash options' do
-    expect_any_instance_of(FormatParser::AIFFParser).to receive(:call).and_return(:audio)
-    result = FormatParser.parse_http('http://localhost:9399/PNG/anim.png', results: :all)
+    fake_result = double(nature: :audio, format: :aiff)
+    expect_any_instance_of(FormatParser::AIFFParser).to receive(:call).and_return(fake_result)
+    results = FormatParser.parse_http('http://localhost:9399/PNG/anim.png', results: :all)
 
-    expect(result.include?(:audio)).to be true
-    expect(result.count).to eq(2)
+    expect(results.count).to eq(2)
+    expect(results).to include(fake_result)
   end
 
   it 'parses the animated PNG over HTTP' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-16 00:00:00.000000000 Z
+date: 2018-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -152,6 +152,7 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - CONTRIBUTING.md
 - Gemfile
@@ -169,6 +170,7 @@ files:
 - lib/image.rb
 - lib/io_constraint.rb
 - lib/io_utils.rb
+- lib/measurometer.rb
 - lib/parsers/aiff_parser.rb
 - lib/parsers/cr2_parser.rb
 - lib/parsers/dpx_parser.rb
@@ -200,6 +202,7 @@ files:
 - spec/file_information_spec.rb
 - spec/format_parser_spec.rb
 - spec/io_utils_spec.rb
+- spec/measurometer_spec.rb
 - spec/parsers/aiff_parser_spec.rb
 - spec/parsers/cr2_parser_spec.rb
 - spec/parsers/dpx_parser_spec.rb