format_parser 0.13.6 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 236f35fe657e5bb8f51cf08724fb3138f17b6a20605af4131a7643711f43cd93
-  data.tar.gz: 65037da607c406be2bf0d8e7eb549537199a4ed8c97243c68b63c62e20bdb9e5
+  metadata.gz: 0d9daf0603ac099b75a9ddb85b8061190dc8fe2f3aad46633ff94b1f4a99020e
+  data.tar.gz: d4343aa08f9ec6a6864a5aed3b7ea174f779c3ea0954417e9b31a2c871126f27
 SHA512:
-  metadata.gz: aaa8a5c25a9b9b6884e0ec22adf90390aa2a32e4268e7e1d01c98a5d88a20bb25a4635ca53fd891d28bae8de34287ff57c943af76c905425f41ade98985408d9
-  data.tar.gz: 54a417ead7b3d12d585f6775feee2295d95f40fe9ac66a7a5e0788dfd6791271d95ddc1ef230a7c3e17c04635765916090db2a0cf98a722e5c7d662f80eb7d37
+  metadata.gz: 2d63a7d9802157e35260b91f5a8077008fa4e7c19837cc4a01ff0d3954fa22d29455a13cb914bc892782d986a7d85b9465eb21b8c753d16c0da774ddb0f5c47c
+  data.tar.gz: 651d41396efdeb4e9f74173d284617baa0bf34f3ce6e8a6890a1ceadf2ef4a3556a3dd0b4b282dc84ea9f65c020ecbe1ae942da78eed85bed6d969161dbb0cbb

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.14.0
+* PDF: Reduce the PDF parser to the basic binary detection (PDF/not PDF) until we have a better/more robust PDF parser
+* MP3: Fix the byte length of MPEG frames calculation to correctly account for ID3V1 and ID3V2 instead of ID3V1 twice
+* MP3: Remove the workaround for `id3tag` choking on non-matching genre strings (bumps dependency on `id3tag`)
+* Use Measurometer provided by the [measurometer gem](https://rubygems.org/gems/measurometer)
+* Ogg: Add support for the Ogg format
+
 ## 0.13.6
 * Make all reads in the MOOV decoder strict - fail early if reads are improperly sized
 * Disable parsing for `udta` atoms in MP4/MOV since we do not have a good way of parsing them yet

data/README.md

@@ -30,12 +30,13 @@ and [dimensions,](https://github.com/sstephenson/dimensions) borrowing from them
 * M4A
 * ZIP
 * DOCX, PPTX, XLSX
+* OGG
 
 ...with [more](https://github.com/WeTransfer/format_parser/issues?q=is%3Aissue+is%3Aopen+label%3Aformats) on the way!
 
 ## Basic usage
 
-Pass an IO object that responds to `read` and `seek` to `FormatParser.parse` and the first confirmed match will be returned.
+Pass an IO object that responds to `read`, `seek` and `size` to `FormatParser.parse` and the first confirmed match will be returned.
 
 ```ruby
 match = FormatParser.parse(File.open("myimage.jpg", "rb"))
@@ -107,6 +108,11 @@ Therefore we adapt the following approaches:
   is easier to verify and test, and we likely don't care about all the metadata anyway
 * Avoid using C libraries which are likely to contain buffer overflows/underflows - we stay memory safe
 
+## Acknowledgements
+
+We are incredibly grateful to Remco van't Veer for [exifr](https://github.com/remvee/exifr) and to
+Krists Ozols for [id3tag](https://github.com/krists/id3tag) that we are using for crucial tasks.
+
 ## Fixture Sources
 
 Unless specified otherwise in this section the fixture files are MIT licensed and from the FastImage and Dimensions projects.
@@ -145,6 +151,9 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
 - atc_fixture_vbr.flac is a converted version of the MP3 with the same name
 - c_11k16btipcm.flac is a converted version of the WAV with the same name
 
+### OGG
+- `hi.ogg`,  `vorbis.ogg`, `with_confusing_magic_string.ogg`, `with_garbage_at_the_end.ogg` have been generated by the project contributors
+
 ### M4A
 - fixture.m4a was created by one of the project maintainers and is MIT licensed
 

data/format_parser.gemspec

@@ -32,8 +32,9 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'ks', '~> 0.0.1'
   spec.add_dependency 'exifr', '~> 1.0'
-  spec.add_dependency 'id3tag', '~> 0.10'
+  spec.add_dependency 'id3tag', '~> 0.10', '>= 0.10.1'
   spec.add_dependency 'faraday', '~> 0.13'
+  spec.add_dependency 'measurometer', '~> 1'
 
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rake', '~> 12'

data/lib/care.rb

@@ -173,10 +173,10 @@ class Care
     # @param io[IO] the IO to read from
     # @param page_i[Integer] which page (zero-based) to read
     def read_page(io, page_i)
-      FormatParser::Measurometer.increment_counter('format_parser.parser.Care.page_reads_from_upsteam', 1)
+      Measurometer.increment_counter('format_parser.parser.Care.page_reads_from_upsteam', 1)
 
       io.seek(page_i * @page_size)
-      read_result = io.read(@page_size)
+      read_result = Measurometer.instrument('format_parser.Care.read_page') { io.read(@page_size) }
       if read_result.nil?
         # If the read went past the end of the IO the read result will be nil,
         # so we know our IO is exhausted here

data/lib/format_parser.rb

@@ -1,4 +1,5 @@
 require 'set'
+require 'measurometer'
 
 # A pretty nimble module for parsing file metadata using partial reads. Contains all the
 # top-level methods of the library.
@@ -17,6 +18,10 @@ module FormatParser
   require_relative 'io_constraint'
   require_relative 'care'
 
+  # Define Measurometer in the internal namespace as well
+  # so that we stay compatible for the applications that use it
+  const_set(:Measurometer, ::Measurometer)
+
   # Is used to manage access to the shared array of parser constructors, which might
   # potentially be mutated from different threads. The mutex won't be hit too often
   # since it only locks when adding/removing parsers.
@@ -95,10 +100,9 @@ module FormatParser
   # @param formats[Array] an array of file formats to scope the parsing to.
   #   For example `[:jpg, :tif]` will scope the parsing to TIFF and JPEG files.
   #   The default value is "all formats known to FormatParser"
-  # @param results[:first, :all, Integer] one of the values defining how many results to return if parsing
-  #   is ambiguous. The default is `:first` which returns the first matching result. Other
-  #   possible values are `:all` to get all possible results and an Integer to return
-  #   at most N results.
+  # @param results[:first, :all] one of the values defining how many results to return if parsing
+  #   is ambiguous. The default is `:first` which returns the first matching result. `:all` will return all results.
+  #   When using `:first` parsing will stop at the first successful match and other parsers won't run.
   # @param limits_config[ReadLimitsConfig] the configuration object for various read/cache limits. The default
   #   one should be good for most cases.
   # @return [Array<Result>, Result, nil] either an Array of results, a single parsing result or `nil`if
@@ -251,7 +255,4 @@ module FormatParser
   Dir.glob(__dir__ + '/parsers/*.rb').sort.each do |parser_file|
     require parser_file
   end
-  # The Measurometer latches itself onto existing classes, so load it after
-  # we have loaded all the parsers
-  require_relative 'measurometer'
 end

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.13.6'
+  VERSION = '0.14.0'
 end

data/lib/parsers/exif_parser.rb

@@ -74,7 +74,9 @@ module FormatParser::EXIFParser
   EXIFR.logger = Logger.new(nil)
 
   def exif_from_tiff_io(constrained_io)
-    raw_exif_data = EXIFR::TIFF.new(IOExt.new(constrained_io))
-    raw_exif_data ? EXIFResult.new(raw_exif_data) : nil
+    Measurometer.instrument('format_parser.EXIFParser.exif_from_tiff_io') do
+      raw_exif_data = EXIFR::TIFF.new(IOExt.new(constrained_io))
+      raw_exif_data ? EXIFResult.new(raw_exif_data) : nil
+    end
   end
 end

data/lib/parsers/jpeg_parser.rb

@@ -60,7 +60,7 @@ class FormatParser::JPEGParser
       end
     end
 
-    FormatParser::Measurometer.add_distribution_value('format_parser.JPEGParser.bytes_read_until_capture', @buf.pos)
+    Measurometer.add_distribution_value('format_parser.JPEGParser.bytes_read_until_capture', @buf.pos)
 
     # Return at the earliest possible opportunity
     if @width && @height
@@ -137,7 +137,7 @@ class FormatParser::JPEGParser
     # ...and only then read the marker contents and parse it as EXIF
     exif_data = safe_read(@buf, app1_frame_content_length - EXIF_MAGIC_STRING.bytesize)
 
-    FormatParser::Measurometer.add_distribution_value('format_parser.JPEGParser.bytes_sent_to_exif_parser', exif_data.bytesize)
+    Measurometer.add_distribution_value('format_parser.JPEGParser.bytes_sent_to_exif_parser', exif_data.bytesize)
 
     @exif_data = exif_from_tiff_io(StringIO.new(exif_data))
   rescue EXIFR::MalformedTIFF

data/lib/parsers/moov_parser.rb

@@ -27,7 +27,9 @@ class FormatParser::MOOVParser
     # size that gets parsed just before.
     max_read_offset = 0xFFFFFFFF
     decoder = Decoder.new
-    atom_tree = decoder.extract_atom_stream(io, max_read_offset)
+    atom_tree = Measurometer.instrument('format_parser.Decoder.extract_atom_stream') do
+      decoder.extract_atom_stream(io, max_read_offset)
+    end
 
     ftyp_atom = decoder.find_first_atom_by_path(atom_tree, 'ftyp')
     file_type = ftyp_atom.field_value(:major_brand)

data/lib/parsers/mp3_parser.rb

@@ -43,10 +43,7 @@ class FormatParser::MP3Parser
     def to_h
       tag = __getobj__
       MEMBERS.each_with_object({}) do |k, h|
-        # ID3Tag sometimes raises when trying to find an unknown genre.
-        # If this guard is removed, it fails when trying to do a gsub on a nil,
-        # in /lib/id3tag/frames/v2/genre_frame/genre_parser_pre_24.rb:25:in `just_genres'
-        value = tag.public_send(k) rescue nil
+        value = tag.public_send(k)
         h[k] = value if value
       end
     end
@@ -74,7 +71,7 @@ class FormatParser::MP3Parser
     # Compute how many bytes are occupied by the actual MPEG frames
     ignore_bytes_at_tail = id3v1 ? 128 : 0
     ignore_bytes_at_head = io.pos
-    bytes_used_by_frames = io.size - ignore_bytes_at_tail - ignore_bytes_at_tail
+    bytes_used_by_frames = io.size - ignore_bytes_at_head - ignore_bytes_at_tail
 
     io.seek(ignore_bytes_at_head)
 

data/lib/parsers/ogg_parser.rb

@@ -0,0 +1,218 @@
+# https://xiph.org/vorbis/doc/Vorbis_I_spec.pdf
+# https://en.wikipedia.org/wiki/Ogg#Page_structure
+class FormatParser::OggParser
+  include FormatParser::IOUtils
+
+  # Maximum size of an Ogg page
+  MAX_POSSIBLE_PAGE_SIZE = 65307
+
+  def call(io)
+    # The format consists of chunks of data each called an "Ogg page". Each page
+    # begins with the characters, "OggS", to identify the file as Ogg format.
+    capture_pattern = safe_read(io, 4)
+    return unless capture_pattern == 'OggS'
+
+    io.seek(28) # skip not important bytes
+
+    # Each header packet begins with the same header fields.
+    #   1) packet_type: 8 bit value (the identification header is type 1)
+    #   2) the characters v','o','r','b','i','s' as six octets
+    packet_type, vorbis, _vorbis_version, channels, sample_rate = safe_read(io, 16).unpack('Ca6VCV')
+    return unless packet_type == 1 && vorbis == 'vorbis'
+
+    # In order to calculate the audio duration we have to read a
+    # granule_position of the last Ogg page of the file. Unfortunately, we don't
+    # know where the last page starts. But we do know that max size of an Ogg
+    # page is 65307 bytes. So we read the last 65307 bytes from the file and try
+    # to find the last page in this tail.
+    pos = io.size - MAX_POSSIBLE_PAGE_SIZE
+    pos = 0 if pos < 0
+    io.seek(pos)
+    tail = io.read(MAX_POSSIBLE_PAGE_SIZE)
+    return unless tail
+
+    granule_position = find_last_granule_position(tail)
+    return unless granule_position
+
+    duration = granule_position / sample_rate.to_f
+    return if duration == Float::INFINITY
+
+    FormatParser::Audio.new(
+      format: :ogg,
+      audio_sample_rate_hz: sample_rate,
+      num_audio_channels: channels,
+      media_duration_seconds: duration
+    )
+  end
+
+  private
+
+  def all_indices_of_substr_in_str(of_substring, in_string)
+    last_i = 0
+    found_at_indices = []
+    while last_i = in_string.index(of_substring, last_i)
+      found_at_indices << last_i
+      last_i += of_substring.bytesize
+    end
+    found_at_indices
+  end
+
+  # Returns granule_position of the last valid Ogg page contained in the given
+  # tail. Since the tail may contain multiple "OggS" entries the method searches
+  # them recursively starting from the end. The search stops when the first
+  # valid Oggs page is found.
+  #
+  # The granule position contains the offset of the page in terms of the
+  # number of samples from the start of file. So once we know that number
+  # we can estimate how long the file is. We _do_ need to add the number
+  # of samples the granule covers though
+  def find_last_granule_position(in_string)
+    # The Ogg page always starts with "OggS". Find all of them
+    # in the given tail, since we want to scan "tail to head" -
+    # starting with the last index and going down to the first
+    rev_indices = all_indices_of_substr_in_str('OggS', in_string).reverse
+    rev_indices.each do |idx|
+      if granule_pos = extract_granule_position_from_string_at(in_string, idx)
+        return granule_pos
+      end
+    end
+    nil # Nothing matched or the list of indices was empty
+  end
+
+  # Since the magic bits may occur inside the body of the page we have to
+  # validate that what we found is actually an Ogg page by calculating the
+  # checksum. For this reason we have to read the entire page and calculate
+  # its checksum. In order to read the entire Ogg page we first have to read a
+  # part of its header to find out the size of the page.
+  def extract_granule_position_from_string_at(string, at)
+    header_size = 27
+    header_bytes = string.byteslice(at, header_size)
+    return unless header_bytes && header_bytes.bytesize == header_size
+
+    # Read the Ogg page header excluding the segment table (in other words read
+    # first 27 bytes). See https://en.wikipedia.org/wiki/Ogg#Page_structure
+    _capture_pattern,
+    _version,
+    _header_type,
+    granule_position,
+    _bitstream_serial_number,
+    _page_sequence_number,
+    checksum,
+    num_bytes_page_segments = header_bytes.unpack('a4CCQ<VVVC')
+
+    # Read the segment table part of the Ogg page header. Its size is stored in page_segments.
+    #
+    # The segment table is a vector of 8-bit values, each indicating the length
+    # of the corresponding segment within the page body.
+    # If there are no segments in the segment table the page is certainly invalid
+    return if num_bytes_page_segments == 0
+
+    # Read the segment table
+    segment_table_pos = at + header_size
+    segment_table = string.byteslice(segment_table_pos, num_bytes_page_segments)
+    return unless segment_table && segment_table.bytesize == num_bytes_page_segments
+
+    # Calculate the size of the Ogg page
+    num_bytes_used_for_segments = segment_table.unpack('C*').inject(&:+)
+    page_size = header_size + num_bytes_page_segments + num_bytes_used_for_segments
+
+    # Read the entire page now that we know how much we have to read
+    entire_page = string.byteslice(at, page_size)
+    return unless entire_page && entire_page.bytesize == page_size
+
+    # Compute and check the checksum. If this check fails it means one of the two:
+    #   - the data is corrupted
+    #   - the "OggS" capture pattern occures inside the body of the page and is
+    #     we were scanning a random piece of content which was not an Ogg page
+    return unless checksum == calculate_checksum(entire_page)
+
+    # ...and only having gone through all these motions - return the granule position.
+    granule_position
+  end
+
+  # Calculate the CRC using the 0x04C11DB7 polynomial. We cannot use Zlib since
+  # it generates different checksums. Copied from https://github.com/anibali/ruby-ogg
+  def calculate_checksum(data)
+    crc_reg = 0
+    data.each_byte.with_index do |byte, i|
+      # The checksum is calculated over _the entire page_ but with the
+      # placeholder for the checksum - the 4 bytes - zeroed out. The checksum
+      # is then substituted _into_ the page at that offset. So when we go
+      # over bytes at these offsets we will substitute them with 0s
+      b = (22..25).cover?(i) ? 0 : byte
+      crc_reg = (crc_reg << 8) ^ CRC_LOOKUP[((crc_reg >> 24) & 0xff) ^ b]
+      crc_reg = crc_reg % 2**32
+    end
+
+    crc_reg
+  end
+
+  CRC_LOOKUP = [
+    0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9,
+    0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005,
+    0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61,
+    0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd,
+    0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9,
+    0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75,
+    0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011,
+    0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd,
+    0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039,
+    0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5,
+    0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81,
+    0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d,
+    0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49,
+    0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95,
+    0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1,
+    0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d,
+    0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae,
+    0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072,
+    0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16,
+    0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca,
+    0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde,
+    0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02,
+    0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066,
+    0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba,
+    0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e,
+    0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692,
+    0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6,
+    0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a,
+    0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e,
+    0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2,
+    0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686,
+    0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a,
+    0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637,
+    0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb,
+    0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f,
+    0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53,
+    0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47,
+    0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b,
+    0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff,
+    0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623,
+    0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7,
+    0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b,
+    0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f,
+    0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3,
+    0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7,
+    0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b,
+    0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f,
+    0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3,
+    0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640,
+    0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c,
+    0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8,
+    0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24,
+    0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30,
+    0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec,
+    0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088,
+    0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654,
+    0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0,
+    0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c,
+    0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18,
+    0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4,
+    0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0,
+    0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c,
+    0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668,
+    0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4
+  ].freeze
+
+  FormatParser.register_parser self, natures: :audio, formats: :ogg
+end

data/lib/parsers/pdf_parser.rb

@@ -9,67 +9,12 @@ class FormatParser::PDFParser
   #
   PDF_MARKER = /%PDF-1\.[0-8]{1}/
 
-  # Page counts have different markers depending on
-  # the PDF type. There is not a single common way of solving
-  # this. The only way of solving this correctly is by adding
-  # different types of PDF's in the specs.
-  #
-  COUNT_MARKERS = ['Count ']
-  EOF_MARKER    = '%EOF'
-
   def call(io)
     io = FormatParser::IOConstraint.new(io)
 
     return unless safe_read(io, 9) =~ PDF_MARKER
 
-    attributes = scan_for_attributes(io)
-
-    FormatParser::Document.new(
-      format: :pdf,
-      page_count: attributes[:page_count]
-    )
-  end
-
-  private
-
-  # Read ahead bytes until one of % or / is reached.
-  # A header in a PDF always starts with a /
-  # The % is to detect the EOF
-  #
-  def scan_for_attributes(io)
-    result = {}
-
-    while read = safe_read(io, 1)
-      case read
-      when '%'
-        break if safe_read(io, EOF_MARKER.size) == EOF_MARKER
-      when '/'
-        find_page_count(io, result)
-      end
-    end
-
-    result
-  end
-
-  def find_page_count(io, result)
-    COUNT_MARKERS.each do |marker|
-      if safe_read(io, marker.size) == marker
-        result[:page_count] = read_numbers(io)
-      end
-    end
-  end
-
-  # Read ahead bytes until no more numbers are found
-  # This assumes that the position of io starts at a
-  # number
-  def read_numbers(io)
-    numbers = ''
-
-    while c = safe_read(io, 1)
-      c =~ /\d+/ ? numbers << c : break
-    end
-
-    numbers.to_i
+    FormatParser::Document.new(format: :pdf)
   end
 
   FormatParser.register_parser self, natures: :document, formats: :pdf

data/lib/read_limiter.rb

@@ -77,9 +77,9 @@ class FormatParser::ReadLimiter
   #   `format_parser.TIFF.read_limiter.num_seeks` and so forth
   # @return void
   def send_metrics(prefix)
-    FormatParser::Measurometer.add_distribution_value('format_parser.%s.read_limiter.num_seeks' % prefix, @seeks)
-    FormatParser::Measurometer.add_distribution_value('format_parser.%s.read_limiter.num_reads' % prefix, @reads)
-    FormatParser::Measurometer.add_distribution_value('format_parser.%s.read_limiter.read_bytes' % prefix, @bytes)
+    Measurometer.add_distribution_value('format_parser.%s.read_limiter.num_seeks' % prefix, @seeks)
+    Measurometer.add_distribution_value('format_parser.%s.read_limiter.num_reads' % prefix, @reads)
+    Measurometer.add_distribution_value('format_parser.%s.read_limiter.read_bytes' % prefix, @bytes)
   end
 
   # Resets all the recorded call counters so that the object can be reused for the next parser,

data/lib/remote_io.rb

@@ -60,7 +60,7 @@ class FormatParser::RemoteIO
   # @return [String] the read bytes
   def read(n_bytes)
     http_range = (@pos..(@pos + n_bytes - 1))
-    maybe_size, maybe_body = request_range(http_range)
+    maybe_size, maybe_body = Measurometer.instrument('format_parser.RemoteIO.read') { request_range(http_range) }
     if maybe_size && maybe_body
       @remote_size = maybe_size
       @pos += maybe_body.bytesize
@@ -103,10 +103,10 @@ class FormatParser::RemoteIO
       # cannot hint size with this response - at lease not when working with S3
       return
     when 500..599
-      FormatParser::Measurometer.increment_counter('format_parser.RemoteIO.upstream50x_errors', 1)
+      Measurometer.increment_counter('format_parser.RemoteIO.upstream50x_errors', 1)
       raise IntermittentFailure.new(response.status, "Server at #{@uri} replied with a #{response.status} and we might want to retry")
     else
-      FormatParser::Measurometer.increment_counter('format_parser.RemoteIO.invalid_request_errors', 1)
+      Measurometer.increment_counter('format_parser.RemoteIO.invalid_request_errors', 1)
       raise InvalidRequest.new(response.status, "Server at #{@uri} replied with a #{response.status} and refused our request")
     end
   end

data/spec/format_parser_spec.rb

@@ -5,6 +5,10 @@ describe FormatParser do
     expect(FormatParser::VERSION).to be_kind_of(String)
   end
 
+  it 'exposes the Measurometer constant' do
+    expect(FormatParser::Measurometer).to be_kind_of(Module)
+  end
+
   describe '.parse' do
     it 'returns nil when trying to parse an empty IO' do
       d = StringIO.new('')

data/spec/parsers/mp3_parser_spec.rb

@@ -57,7 +57,7 @@ describe FormatParser::MP3Parser do
     expect(parsed.format).to eq(:mp3)
     expect(parsed.num_audio_channels).to eq(2)
     expect(parsed.audio_sample_rate_hz).to eq(44100)
-    expect(parsed.media_duration_seconds).to be_within(0.1).of(1102.46)
+    expect(parsed.media_duration_seconds).to be_within(0.1).of(1098.03)
 
     expect(parsed.intrinsics).not_to be_nil
 

data/spec/parsers/ogg_parser_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+describe FormatParser::OggParser do
+  it 'parses an ogg file' do
+    parse_result = subject.call(File.open(__dir__ + '/../fixtures/Ogg/vorbis.ogg', 'rb'))
+
+    expect(parse_result.nature).to eq(:audio)
+    expect(parse_result.format).to eq(:ogg)
+    expect(parse_result.num_audio_channels).to eq(1)
+    expect(parse_result.audio_sample_rate_hz).to eq(16000)
+    expect(parse_result.media_duration_seconds).to be_within(0.01).of(2973.95)
+  end
+
+  it 'skips a file if it contains more than MAX_POSSIBLE_OGG_PAGE_SIZE bytes of garbage at the end' do
+    parse_result = subject.call(File.open(__dir__ + '/../fixtures/Ogg/with_garbage_at_the_end.ogg', 'rb'))
+    expect(parse_result).to be_nil
+  end
+
+  it "correctly parses an ogg file when a magic string occurs in the page's body" do
+    parse_result = subject.call(File.open(__dir__ + '/../fixtures/Ogg/with_confusing_magic_string.ogg', 'rb'))
+
+    expect(parse_result.nature).to eq(:audio)
+    expect(parse_result.format).to eq(:ogg)
+    expect(parse_result.num_audio_channels).to eq(1)
+    expect(parse_result.audio_sample_rate_hz).to eq(8000)
+    expect(parse_result.media_duration_seconds).to be_within(0.01).of(0.45)
+  end
+end

data/spec/parsers/pdf_parser_spec.rb

@@ -18,10 +18,6 @@ describe FormatParser::PDFParser do
       expect(parsed_pdf.nature).to eq(:document)
       expect(parsed_pdf.format).to eq(:pdf)
     end
-
-    it 'has a correct page count' do
-      expect(parsed_pdf.page_count).to eq(hash.fetch(:page_count))
-    end
   end
 
   describe 'a PDF file with a missing version header' do
@@ -44,25 +40,9 @@ describe FormatParser::PDFParser do
     pending 'does not parse succesfully'
   end
 
-  describe 'a PDF file with a missing COUNT_HEADER' do
-    let(:pdf_file) { 'missing_page_count.pdf' }
-
-    it 'does not return a page count' do
-      expect(parsed_pdf.page_count).to eq(nil)
-    end
-  end
-
   describe 'parses a PDF file' do
     describe 'a single page file' do
-      include_examples :behave_like_pdf, file: '1_page.pdf', page_count: 1
-    end
-
-    describe 'a multi page pdf file' do
-      include_examples :behave_like_pdf, file: '2_pages.pdf', page_count: 2
-    end
-
-    describe 'a multi page pdf file with content' do
-      include_examples :behave_like_pdf, file: '10_pages.pdf', page_count: 10
+      include_examples :behave_like_pdf, file: '1_page.pdf'
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.13.6
+  version: 0.14.0
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-05-28 00:00:00.000000000 Z
+date: 2018-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -46,6 +46,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -53,6 +56,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.1
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -67,6 +73,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.13'
+- !ruby/object:Gem::Dependency
+  name: measurometer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -186,7 +206,6 @@ files:
 - lib/image.rb
 - lib/io_constraint.rb
 - lib/io_utils.rb
-- lib/measurometer.rb
 - lib/parsers/aiff_parser.rb
 - lib/parsers/bmp_parser.rb
 - lib/parsers/cr2_parser.rb
@@ -201,6 +220,7 @@ files:
 - lib/parsers/moov_parser/decoder.rb
 - lib/parsers/mp3_parser.rb
 - lib/parsers/mp3_parser/id3_extraction.rb
+- lib/parsers/ogg_parser.rb
 - lib/parsers/pdf_parser.rb
 - lib/parsers/png_parser.rb
 - lib/parsers/psd_parser.rb
@@ -220,7 +240,6 @@ files:
 - spec/format_parser_inspect_spec.rb
 - spec/format_parser_spec.rb
 - spec/io_utils_spec.rb
-- spec/measurometer_spec.rb
 - spec/parsers/aiff_parser_spec.rb
 - spec/parsers/bmp_parser_spec.rb
 - spec/parsers/cr2_parser_spec.rb
@@ -232,6 +251,7 @@ files:
 - spec/parsers/jpeg_parser_spec.rb
 - spec/parsers/moov_parser_spec.rb
 - spec/parsers/mp3_parser_spec.rb
+- spec/parsers/ogg_parser_spec.rb
 - spec/parsers/pdf_parser_spec.rb
 - spec/parsers/png_parser_spec.rb
 - spec/parsers/psd_parser_spec.rb

data/lib/measurometer.rb

@@ -1,100 +0,0 @@
-class FormatParser::Measurometer
-  class << self
-    # Permits adding instrumentation drivers. Measurometer is 1-1 API
-    # compatible with Appsignal, which we use a lot. So to magically
-    # obtain all Appsignal instrumentation, add the Appsignal module
-    # as a driver.
-    #
-    #   Measurometer.drivers << Appsignal
-    #
-    # A driver must be reentrant and thread-safe - it should be possible
-    # to have multiple `instrument` calls open from different threads at the
-    # same time.
-    # The driver must support the same interface as the Measurometer class
-    # itself, minus the `drivers` and `instrument_instance_method` methods.
-    #
-    # @return Array
-    def drivers
-      @drivers ||= []
-      @drivers
-    end
-
-    # Runs a given block within a cascade of `instrument` blocks of all the
-    # added drivers.
-    #
-    #   Measurometer.instrument('do_foo') { compute! }
-    #
-    # unfolds to
-    #   Appsignal.instrument('do_foo') do
-    #     Statsd.timing('do_foo') do
-    #       compute!
-    #     end
-    #   end
-    #
-    # A driver must be reentrant and thread-safe - it should be possible
-    # to have multiple `instrument` calls open from different threads at the
-    # same time.
-    # The driver must support the same interface as the Measurometer class
-    # itself, minus the `drivers` and `instrument_instance_method` methods.
-    #
-    # @param block_name[String] under which path to push the metric
-    # @param blk[#call] the block to instrument
-    # @return [Object] the return value of &blk
-    def instrument(block_name, &blk)
-      return yield unless @drivers && @drivers.any? # The block wrapping business is not free
-      @drivers.inject(blk) { |outer_block, driver|
-        -> {
-          driver.instrument(block_name, &outer_block)
-        }
-      }.call
-    end
-
-    # Adds a distribution value (sample) under a given path
-    #
-    # @param value_path[String] under which path to push the metric
-    # @param value[Numeric] distribution value
-    # @return nil
-    def add_distribution_value(value_path, value)
-      (@drivers || []).each { |d| d.add_distribution_value(value_path, value) }
-      nil
-    end
-
-    # Increment a named counter under a given path
-    #
-    # @param counter_path[String] under which path to push the metric
-    # @param by[Integer] the counter increment to apply
-    # @return nil
-    def increment_counter(counter_path, by)
-      (@drivers || []).each { |d| d.increment_counter(counter_path, by) }
-      nil
-    end
-
-    # Wrap an anonymous module around an instance method in the given class to have
-    # it instrumented automatically. The name of the measurement will be interpolated as:
-    #
-    #   "#{prefix}.#{rightmost_class_constant_name}.#{instance_method_name}"
-    #
-    # @param target_class[Class] the class to instrument
-    # @param instance_method_name_to_instrument[Symbol] the method name to instrument
-    # @param path_prefix[String] under which path to push the instrumented metric
-    # @return void
-    def instrument_instance_method(target_class, instance_method_name_to_instrument, path_prefix)
-      short_class_name = target_class.to_s.split('::').last
-      instrumentation_name = [path_prefix, short_class_name, instance_method_name_to_instrument].join('.')
-      instrumenter_module = Module.new do
-        define_method(instance_method_name_to_instrument) do |*any|
-          ::FormatParser::Measurometer.instrument(instrumentation_name) { super(*any) }
-        end
-      end
-      target_class.prepend(instrumenter_module)
-    end
-  end
-
-  # Instrument things interesting in the global sense
-  instrument_instance_method(FormatParser::RemoteIO, :read, 'format_parser')
-  instrument_instance_method(Care::Cache, :read_page, 'format_parser')
-
-  # Instrument more specific things on a per-parser basis
-  instrument_instance_method(FormatParser::EXIFParser, :scan_image_tiff, 'format_parser')
-  instrument_instance_method(FormatParser::MOOVParser::Decoder, :extract_atom_stream, 'format_parser.parsers.MOOVParser')
-end

data/spec/measurometer_spec.rb

@@ -1,48 +0,0 @@
-require 'spec_helper'
-
-describe FormatParser::Measurometer do
-  RSpec::Matchers.define :include_counter_or_measurement_named do |named|
-    match do |actual|
-      actual.any? do |e|
-        e[0] == named && e[1] > 0
-      end
-    end
-  end
-
-  it 'instruments a full cycle FormatParser.parse' do
-    driver_class = Class.new do
-      attr_accessor :timings, :counters, :distributions
-      def instrument(block_name)
-        s = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-        yield.tap do
-          delta = Process.clock_gettime(Process::CLOCK_MONOTONIC) - s
-          @timings ||= []
-          @timings << [block_name, delta * 1000]
-        end
-      end
-
-      def add_distribution_value(value_path, value)
-        @distributions ||= []
-        @distributions << [value_path, value]
-      end
-
-      def increment_counter(value_path, value)
-        @counters ||= []
-        @counters << [value_path, value]
-      end
-    end
-
-    instrumenter = driver_class.new
-    described_class.drivers << instrumenter
-
-    FormatParser.parse(File.open(fixtures_dir + 'JPEG/keynote_recognized_as_jpeg.key', 'rb'), results: :all)
-
-    described_class.drivers.delete(instrumenter)
-    expect(described_class.drivers).not_to include(instrumenter)
-
-    expect(instrumenter.counters).to include_counter_or_measurement_named('format_parser.detected_formats.zip')
-    expect(instrumenter.counters).to include_counter_or_measurement_named('format_parser.parser.Care.page_reads_from_upsteam')
-    expect(instrumenter.distributions).to include_counter_or_measurement_named('format_parser.ZIPParser.read_limiter.read_bytes')
-    expect(instrumenter.timings).to include_counter_or_measurement_named('format_parser.Cache.read_page')
-  end
-end