format_parser 0.25.1 → 0.25.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 87adbfef15c2281ab6a13f151b857409f0ffad0ecc5270d9d0bbc5cebe207cdb
-  data.tar.gz: 332e3c4efd4ae01b3cf47c669debba7cc1aee5c264bef503720f632f6d801054
+  metadata.gz: dbc915cd212f2207b6f54cc4b567ea342a0bfd76af1e5e3db23330a6cc879e27
+  data.tar.gz: 0a623b8303cbf40a58d348b4e1adcda2d6fc0170b411fba529f9d3ae53533282
 SHA512:
-  metadata.gz: 231d768d4b69b2c2f29bcb861888a7fbb0f4242eec5c9313d6428c9053b4fb4e7d20b8615d731a695c278ac59d3bf07b4977d217892aa1fce8fa7adc9d415efa
-  data.tar.gz: 732fae92e71f6a25fa98b7d88fa70b82404f00c234a6debd641e20ad8ffd9e45c78270f20adb8ad5593cd815e128a2a409ec639351c985b79763d4bfd821fec9
+  metadata.gz: 0ed65c67f508264c0c6c12e88287266ef2bb9346c7e44934ac2e8ac97001de8f5e7f605bdd497ee6f95a45bcf4ad7f6b69021d25739e85debafaecfca76153ce
+  data.tar.gz: 4b250cd8f41bbba735e5d90a4a829c4fea699f04dd190a37499853ed48475aecd19f6d32360f081474355a4f5f9c562e1ca78e8c7c8aba91e5fb9e0ca4930299

data/.travis.yml

@@ -2,8 +2,9 @@ rvm:
 - 2.2.10
 - 2.3.8
 - 2.4.9
-- 2.5.7
-- 2.6.5
+- 2.5.8
+- 2.6.6
+- 2.7.2
 - jruby
 sudo: false
 cache: bundler

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## 0.25.6
+* Fix FormatParser.parse (with `results: :first`) to be deterministic
+
+## 0.25.5
+* DPX: Fix DPXParser to support images without aspect ratio
+
+## 0.25.4
+* MP3: Fix MP3Parser to return nil for TIFF files
+* Add support to ruby 2.7
+
+## 0.25.3
+* MP3: Fix parser to not skip the first bytes if it's not an ID3 header
+
+## 0.25.2
+* Hotfix Moov parser
+
 ## 0.25.1
 * MOV: Fix error "negative length"
 * MOV: Fix reading dimensions in multi-track files

data/CONTRIBUTING.md

@@ -83,32 +83,59 @@ of software. Ideally, this file is going to be something you have produced yours
 and you are permitted to share under the MIT license provisions.
 
 When writing a parser, please try to ensure it returns a usable result as soon as possible,
-or no result as soon as possible (once you know the file is not fit for your specific parser).
+or `nil` as soon as possible (once you know the file is not fit for your specific parser).
 Bear in mind that we enforce read budgets per-parser, so you will not be allowed to perform
 too many reads, or perform reads which are too large.
 
-In order to create new parsers, it is recommended to make a well-named class with an instance method `call`.
+In order to create new parsers, make a well-named class with an instance method `call`,
+and to register a single instance of that class as the parser - so that only one object needs to be stored
+in memory when parsing multiple inputs. In that case your object must be **thread-safe and stateless** - this
+is really important since FormatParser is thread-safe and multiple parsing procedures may be in progress
+concurrently against the same parser object. You can also create a Proc if your parser is fairly trivial.
 
-`call` accepts the IO-ish object as an argument, parses data that it reads from it,
-and then returns the metadata for the file (if it could recover any) or `nil` if it couldn't. All files pass
-through all parsers by default, so if you are dealing with a file that is not "your" format - return `nil` from
-your method or `break` your Proc as early as possible. A blank `return` works fine too.
+If it will be difficult to have your parser thread-safe you can register your class itself as
+the parser and define the `self.call` method to parse using a fresh instance every time, allowing
+object-level state:
+
+```ruby
+class MyParser
+  def self.call(io)
+    new.call(io)
+  end
+
+  def call(io)
+    @state = ...
+  end
+```
+
+`call` accepts a single argument - an IO-ish object which is guaranteed to respond to the same methods as the
+ones defined in `IOConstraint` - that is, it is a strict subset of a standard Ruby IO object. *All reads from
+this IO object are guaranteed to be returned in binary encoding.* The IO will be at offset of 0 when your parsing
+proc receives it and there will be no concurrent calls to that object until your proc returns.
 
-The IO will at the minimum support the subset of the IO API defined in `IOConstraint`
+Your parsing procedure may read from this IO object, and should return either a `Result`-like object with
+the file metadata (if it could recover any) or `nil` if it couldn't. All files pass
+through all parsers by default, so if you are dealing with a file that is not "your" format - return `nil` from
+your method or `break` your Proc as early as possible. A blank `return` works fine too as it actually returns `nil`.
 
-Your parser has to be registered using `FormatParser.register_parser` with the information on the formats
-and file natures it provides.
+Your parser then needs to be registered using `FormatParser.register_parser` with the information on the formats
+and file natures it provides. This allows FormatParser to skip your parser if, say, the user only want to parse for
+`:image` nature files but your parser parses `:audio`.
 
-Down below you can find the most basic parser implementation:
+Down below you can find the most basic parser implementation which parses an imaginary `IMGA` file format:
 
 ```ruby
 MyParser = ->(io) {
-  # ... do some parsing with `io`
+  # ... Read the magic bytes from the start of IO - the IO is
+  # guaranteed to be fed to you at offset 0, start-of-file.
   magic_bytes = io.read(4)
+
   # breaking the block returns `nil` to the caller signaling "no match"
   break if magic_bytes != 'IMGA'
 
+  # Our file format stores the width and height as 2 32-bit unsigned integers
   parsed_witdh, parsed_height = io.read(8).unpack('VV')
+
   # ...and return the FileInformation::Image object with the metadata.
   FormatParser::Image.new(
     format: :imga,
@@ -135,8 +162,8 @@ class MyParser
     # ... do some parsing with `io`
     # The instance will be discarded after parsing, so using instance variables
     # is permitted - they are not shared between calls to `call`
-    @magic_bytes = io.read(4)
-    break if @magic_bytes != 'IMGA'
+    magic_bytes = io.read(4)
+    break if magic_bytes != 'IMGA'
     parsed_witdh, parsed_height = io.read(8).unpack('VV')
     FormatParser::Image.new(
       format: :imga,
@@ -145,23 +172,33 @@ class MyParser
     )
   end
 
-  FormatParser.register_parser self, natures: :image, formats: :bmp
+  # Note that we register an instance of the class, not the class. It is the
+  # instance that responds to `call()` and we can do this because our object
+  # is stateless.
+  FormatParser.register_parser new, natures: :image, formats: :bmp
 end
 ```
 
-### Calling convention for preparing parsers
+If your parser supports file types which have a known filename extension, you can add a method to it called `likely_match?`,
+add this method on the object you register itself. For example, for the ZIP parser we use:
+
+```ruby
+  def likely_match?(filename)
+    filename =~ /\.(zip|docx|keynote|numbers|pptx|xlsx)$/i
+  end
+```
 
-A parser that gets registered using `register_parser` must be either:
+If your parser matches the filename it is going to be applied *earlier*, saving time. Since most FormatParser users are
+likely to only want the first result of the parsing, the sooner your parser gets applied - the sooner you can return the result,
+avoiding unnecessary reads.
 
-1) An object that can be `call()`-ed itself, with an argument that conforms to `IOConstraint`
-2) An object that responds to `new` and returns something that can be `call()`-ed with with an argument that conforms to `IOConstraint`.
+### Calling convention for preparing parsers
 
-The second opton is recommended  for most cases.
+A parser that gets registered using `register_parser` must be an object that can be `call()`-ed, with an argument that conforms to `IOConstraint`
 
 FormatParser is made to be used in threaded environments, and if you use instance variables
-you need your parser to be isolated from it's siblings in other threads - therefore you can pass
-a Class on registration to have your parser instantiated for each `call()`, anew.
-
+you need your parser to be isolated from it's siblings in other threads - create a copy for one-off use inside
+your `call` method.
 
 ## Pull requests
 

data/lib/format_parser.rb

@@ -36,7 +36,7 @@ module FormatParser
   # Register a parser object to be used to perform file format detection. Each parser FormatParser
   # provides out of the box registers itself using this method.
   #
-  # @param callable_or_responding_to_new[#call, #new] an object that either responds to #new or to #call
+  # @param callable_parser[#call] an object that responds to #call for parsing an IO
   # @param formats[Array<Symbol>] file formats that the parser provides
   # @param natures[Array<Symbol>] file natures that the parser provides
   # @param priority[Integer] whether the parser has to be applied first or later. Parsers that offer the safest
@@ -45,39 +45,41 @@ module FormatParser
   #   with a lower priority value will be applied first, and if a single result is requested, will also return
   #   first.
   # @return void
-  def self.register_parser(callable_or_responding_to_new, formats:, natures:, priority: LEAST_PRIORITY)
+  def self.register_parser(callable_parser, formats:, natures:, priority: LEAST_PRIORITY)
     parser_provided_formats = Array(formats)
     parser_provided_natures = Array(natures)
     PARSER_MUX.synchronize do
-      @parsers ||= Set.new
-      @parsers << callable_or_responding_to_new
+      # It can't be a Set because the method `parsers_for` depends on the order
+      # that the parsers were added.
+      @parsers ||= []
+      @parsers << callable_parser unless @parsers.include?(callable_parser)
       @parsers_per_nature ||= {}
       parser_provided_natures.each do |provided_nature|
         @parsers_per_nature[provided_nature] ||= Set.new
-        @parsers_per_nature[provided_nature] << callable_or_responding_to_new
+        @parsers_per_nature[provided_nature] << callable_parser
       end
       @parsers_per_format ||= {}
       parser_provided_formats.each do |provided_format|
         @parsers_per_format[provided_format] ||= Set.new
-        @parsers_per_format[provided_format] << callable_or_responding_to_new
+        @parsers_per_format[provided_format] << callable_parser
       end
       @parser_priorities ||= {}
-      @parser_priorities[callable_or_responding_to_new] = priority
+      @parser_priorities[callable_parser] = priority
     end
   end
 
   # Deregister a parser object (makes FormatParser forget this parser existed). Is mostly used in
   # tests, but can also be used to forcibly disable some formats completely.
   #
-  # @param callable_or_responding_to_new[#call, #new] an object that either responds to #new or to #call
+  # @param callable_parser[#==] an object that is identity-equal to any other registered parser
   # @return void
-  def self.deregister_parser(callable_or_responding_to_new)
+  def self.deregister_parser(callable_parser)
     # Used only in tests
     PARSER_MUX.synchronize do
-      (@parsers || []).delete(callable_or_responding_to_new)
-      (@parsers_per_nature || {}).values.map { |e| e.delete(callable_or_responding_to_new) }
-      (@parsers_per_format || {}).values.map { |e| e.delete(callable_or_responding_to_new) }
-      (@parser_priorities || {}).delete(callable_or_responding_to_new)
+      (@parsers || []).delete(callable_parser)
+      (@parsers_per_nature || {}).values.map { |e| e.delete(callable_parser) }
+      (@parsers_per_format || {}).values.map { |e| e.delete(callable_parser) }
+      (@parser_priorities || {}).delete(callable_parser)
     end
   end
 
@@ -255,7 +257,19 @@ module FormatParser
     # Order the parsers according to their priority value. The ones having a lower
     # value will sort higher and will be applied sooner
     parsers_in_order_of_priority = parsers.to_a.sort do |parser_a, parser_b|
-      @parser_priorities[parser_a] <=> @parser_priorities[parser_b]
+      if @parser_priorities[parser_a] != @parser_priorities[parser_b]
+        @parser_priorities[parser_a] <=> @parser_priorities[parser_b]
+      else
+        # Some parsers have the same priority and we want them to be always sorted
+        # in the same way, to not change the result of FormatParser.parse(results: :first).
+        # When this changes, it can generate flaky tests or event different
+        # results in different environments, which can be hard to understand why.
+        # There is also no guarantee in the order that the elements are added in
+        # @@parser_priorities
+        # So, to have always the same order, we sort by the order that the parsers
+        # were registered if the priorities are the same.
+        @parsers.index(parser_a) <=> @parsers.index(parser_b)
+      end
     end
 
     # If there is one parser that is more likely to match, place it first

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.25.1'
+  VERSION = '0.25.6'
 end

data/lib/parsers/dpx_parser.rb

@@ -35,18 +35,23 @@ class FormatParser::DPXParser
     w = dpx_structure.fetch(:image).fetch(:pixels_per_line)
     h = dpx_structure.fetch(:image).fetch(:lines_per_element)
 
+    display_w = w
+    display_h = h
+
     pixel_aspect_w = dpx_structure.fetch(:orientation).fetch(:horizontal_pixel_aspect)
     pixel_aspect_h = dpx_structure.fetch(:orientation).fetch(:vertical_pixel_aspect)
-    pixel_aspect = pixel_aspect_w / pixel_aspect_h.to_f
 
-    image_aspect = w / h.to_f * pixel_aspect
+    # Find display height and width based on aspect only if the file structure has pixel aspects
+    if pixel_aspect_h != 0 && pixel_aspect_w != 0
+      pixel_aspect = pixel_aspect_w / pixel_aspect_h.to_f
 
-    display_w = w
-    display_h = h
-    if image_aspect > 1
-      display_h = (display_w / image_aspect).round
-    else
-      display_w = (display_h * image_aspect).round
+      image_aspect = w / h.to_f * pixel_aspect
+
+      if image_aspect > 1
+        display_h = (display_w / image_aspect).round
+      else
+        display_w = (display_h * image_aspect).round
+      end
     end
 
     FormatParser::Image.new(

data/lib/parsers/moov_parser/decoder.rb

@@ -68,7 +68,7 @@ class FormatParser::MOOVParser::Decoder
   def find_video_trak_atom(atoms)
     trak_atoms = find_atoms_by_path(atoms, ['moov', 'trak'])
 
-    return [] if trak_atoms.empty?
+    return if trak_atoms.empty?
 
     trak_atoms.find do |trak_atom|
       hdlr_atom = find_first_atom_by_path([trak_atom], 'trak', 'mdia', 'hdlr')

data/lib/parsers/mp3_parser.rb

@@ -29,6 +29,10 @@ class FormatParser::MP3Parser
   ZIP_LOCAL_ENTRY_SIGNATURE = "PK\x03\x04\x14\x00".b
   PNG_HEADER_BYTES = [137, 80, 78, 71, 13, 10, 26, 10].pack('C*')
 
+  MAGIC_LE = [0x49, 0x49, 0x2A, 0x0].pack('C4')
+  MAGIC_BE = [0x4D, 0x4D, 0x0, 0x2A].pack('C4')
+  TIFF_HEADER_BYTES = [MAGIC_LE, MAGIC_BE]
+
   # Wraps the Tag object returned by ID3Tag in such
   # a way that a usable JSON representation gets
   # returned
@@ -68,11 +72,16 @@ class FormatParser::MP3Parser
     return if header.start_with?(ZIP_LOCAL_ENTRY_SIGNATURE)
     return if header.start_with?(PNG_HEADER_BYTES)
 
+    io.seek(0)
+    return if TIFF_HEADER_BYTES.include?(safe_read(io, 4))
+
     # Read all the ID3 tags (or at least attempt to)
     io.seek(0)
     id3v1 = ID3Extraction.attempt_id3_v1_extraction(io)
     tags = [id3v1, ID3Extraction.attempt_id3_v2_extraction(io)].compact
 
+    io.seek(0) if tags.empty?
+
     # Compute how many bytes are occupied by the actual MPEG frames
     ignore_bytes_at_tail = id3v1 ? 128 : 0
     ignore_bytes_at_head = io.pos

data/lib/parsers/tiff_parser.rb

@@ -4,6 +4,7 @@ class FormatParser::TIFFParser
 
   MAGIC_LE = [0x49, 0x49, 0x2A, 0x0].pack('C4')
   MAGIC_BE = [0x4D, 0x4D, 0x0, 0x2A].pack('C4')
+  HEADER_BYTES = [MAGIC_LE, MAGIC_BE]
 
   def likely_match?(filename)
     filename =~ /\.tiff?$/i
@@ -12,7 +13,7 @@ class FormatParser::TIFFParser
   def call(io)
     io = FormatParser::IOConstraint.new(io)
 
-    return unless [MAGIC_LE, MAGIC_BE].include?(safe_read(io, 4))
+    return unless HEADER_BYTES.include?(safe_read(io, 4))
     io.seek(io.pos + 2) # Skip over the offset of the IFD, EXIFR will re-read it anyway
     return if cr2?(io)
 

data/spec/format_parser_spec.rb

@@ -173,6 +173,26 @@ describe FormatParser do
       prioritized_parsers = FormatParser.parsers_for([:archive, :document, :image, :audio], [:tif, :jpg, :zip, :docx, :mp3, :aiff], 'a-file.zip')
       expect(prioritized_parsers.first).to be_kind_of(FormatParser::ZIPParser)
     end
+
+    it 'sorts the parsers by priority and name' do
+      parsers = FormatParser.parsers_for(
+        [:audio, :image],
+        [:cr2, :dpx, :fdx, :flac, :gif, :jpg, :mov, :mp4, :m4a, :mp3, :mpg, :mpeg, :ogg, :png, :tif, :wav]
+      )
+
+      expect(parsers.map { |parser| parser.class.name }).to eq([
+        'FormatParser::GIFParser',
+        'Class',
+        'FormatParser::PNGParser',
+        'FormatParser::CR2Parser',
+        'FormatParser::DPXParser',
+        'FormatParser::FLACParser',
+        'FormatParser::MP3Parser',
+        'FormatParser::OggParser',
+        'FormatParser::TIFFParser',
+        'FormatParser::WAVParser'
+      ])
+    end
   end
 
   describe '.register_parser and .deregister_parser' do

data/spec/parsers/mp3_parser_spec.rb

@@ -166,6 +166,18 @@ describe FormatParser::MP3Parser do
     expect(parsed.artist). to eq('wetransfer')
   end
 
+  it 'does not skip the first bytes if it is not a id3 tag header' do
+    fpath = fixtures_dir + '/MP3/no_id3_tags.mp3'
+
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed).not_to be_nil
+
+    expect(parsed.nature).to eq(:audio)
+    expect(parsed.format).to eq(:mp3)
+    expect(parsed.audio_sample_rate_hz).to eq(44100)
+  end
+
   describe '#as_json' do
     it 'converts all hash keys to string when stringify_keys: true' do
       fpath = fixtures_dir + '/MP3/Cassy.mp3'
@@ -193,4 +205,12 @@ describe FormatParser::MP3Parser do
       ).to eq([ID3Tag::Tag])
     end
   end
+
+  it 'does not recognize TIFF files as MP3' do
+    fpath = fixtures_dir + '/TIFF/test2.tif'
+
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed).to be_nil
+  end
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.25.1
+  version: 0.25.6
 platform: ruby
 authors:
 - Noah Berman
 - Julik Tarkhanov
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-10-02 00:00:00.000000000 Z
+date: 2020-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -277,7 +277,7 @@ licenses:
 - MIT (Hippocratic)
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -292,8 +292,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: A library for efficient parsing of file metadata
 test_files: []