format_parser 0.14.1 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90513000619c11b5a98669f465add204ca68465065681424627adcdf66c634dd
-  data.tar.gz: 1515277ec5edc3bedc7631a7a3e5b56975010e1d7a84b5e166b4eae6977eb90a
+  metadata.gz: 32b5a18eb53b1c5197a828757056354c655225fe7047418a615bfb8f950a6805
+  data.tar.gz: df289b5f4039f89d78b35e481efe50ea95f654497a7783bbaf0a35b997d630ed
 SHA512:
-  metadata.gz: ec36c70f585ebc72089be82bba55b21b83d53d59a4233f33be4e78cbde5f7b12bdb99587238a00cfdf023a24daafadc8803d43e88bacd994c97cfe47c3aac4f1
-  data.tar.gz: 1e964c45d81b43def5575331f02f0949892ddc06f006c7b83275f72cffafefef25cef6b2b8627db0edb6969a8da1e72fa612f600091630fe0822f842062e75d1
+  metadata.gz: 358c318bf9f5262edb1acde1854e5caebaac3e945502d1d164f982f0d8fd26eb717ed5c43b3311f130d6ca2804b8c2bb1c0f1a596f462b01c59cd5a79267b0fb
+  data.tar.gz: 9482a5702896f47ba6da0d99d0e5ef31d51981bdd9271a764872fd06f0a210e19202c1c4fbd111111645d894703646648e8a7f003df7ea2419deef3254ea32da

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.15.0
+* Allow setting `:priority` when registering a parser, to make sure certain parsers are applied earlier - depending
+  on detection confidence and file format popularity at WT.
+
 ## 0.14.1
 * Care caching: Clear pages more deliberately instead of relegating them to GC
 * JPEG: Clear the EXIF buffer explicitly

data/README.md

@@ -157,6 +157,9 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
 ### M4A
 - fixture.m4a was created by one of the project maintainers and is MIT licensed
 
+### PNG
+- `simulator_screenie.png` provided by [Rens Verhoeven](https://github.com/renssies)
+
 ### TIFF
 - `Shinbutsureijoushuincho.tiff` is obtained from Wikimedia Commons and is Creative Commons licensed
 - `IMG_9266_*.tif` and all it's variations were created by the project maintainers

data/lib/format_parser.rb

@@ -28,14 +28,22 @@ module FormatParser
   PARSER_MUX = Mutex.new
   MAX_BYTES_READ_PER_PARSER = 1024 * 1024 * 2
 
+  # The value will ensure the parser having it will be applied to the file last.
+  LEAST_PRIORITY = 99
+
   # Register a parser object to be used to perform file format detection. Each parser FormatParser
   # provides out of the box registers itself using this method.
   #
   # @param callable_or_responding_to_new[#call, #new] an object that either responds to #new or to #call
   # @param formats[Array<Symbol>] file formats that the parser provides
   # @param natures[Array<Symbol>] file natures that the parser provides
+  # @param priority[Integer] whether the parser has to be applied first or later. Parsers that offer the safest
+  #   detection and have the most popular file formats should get a lower priority (0 or 1), the default
+  #   priority is 99. Before parsing parsers get sorted according to their priority value ascending, so parsers
+  #   with a lower priority value will be applied first, and if a single result is requested, will also return
+  #   first.
   # @return void
-  def self.register_parser(callable_or_responding_to_new, formats:, natures:)
+  def self.register_parser(callable_or_responding_to_new, formats:, natures:, priority: LEAST_PRIORITY)
     parser_provided_formats = Array(formats)
     parser_provided_natures = Array(natures)
     PARSER_MUX.synchronize do
@@ -51,6 +59,8 @@ module FormatParser
         @parsers_per_format[provided_format] ||= Set.new
         @parsers_per_format[provided_format] << callable_or_responding_to_new
       end
+      @parser_priorities ||= {}
+      @parser_priorities[callable_or_responding_to_new] = priority
     end
   end
 
@@ -65,6 +75,7 @@ module FormatParser
       (@parsers || []).delete(callable_or_responding_to_new)
       (@parsers_per_nature || {}).values.map { |e| e.delete(callable_or_responding_to_new) }
       (@parsers_per_format || {}).values.map { |e| e.delete(callable_or_responding_to_new) }
+      (@parser_priorities || {}).delete(callable_or_responding_to_new)
     end
   end
 
@@ -229,7 +240,13 @@ module FormatParser
       raise ArgumentError, "No parsers provide both natures #{desired_natures.inspect} and formats #{desired_formats.inspect}"
     end
 
-    factories.map { |callable_or_class| instantiate_parser(callable_or_class) }
+    # Order the parsers according to their priority value. The ones having a lower
+    # value will sort higher and will be applied sooner
+    factories_in_order_of_priority = factories.to_a.sort do |parser_factory_a, parser_factory_b|
+      @parser_priorities[parser_factory_a] <=> @parser_priorities[parser_factory_b]
+    end
+
+    factories_in_order_of_priority.map { |callable_or_class| instantiate_parser(callable_or_class) }
   end
 
   # Instantiates a parser object (an object that responds to `#call`) from a given class

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.14.1'
+  VERSION = '0.15.0'
 end

data/lib/parsers/gif_parser.rb

@@ -44,5 +44,5 @@ class FormatParser::GIFParser
     )
   end
 
-  FormatParser.register_parser self, natures: :image, formats: :gif
+  FormatParser.register_parser self, natures: :image, formats: :gif, priority: 0
 end

data/lib/parsers/jpeg_parser.rb

@@ -163,5 +163,5 @@ class FormatParser::JPEGParser
     safe_skip(@buf, length)
   end
 
-  FormatParser.register_parser self, natures: :image, formats: :jpg
+  FormatParser.register_parser self, natures: :image, formats: :jpg, priority: 0
 end

data/lib/parsers/moov_parser.rb

@@ -83,5 +83,5 @@ class FormatParser::MOOVParser
     maybe_atom_size >= minimum_ftyp_atom_size && maybe_ftyp_atom_signature == 'ftyp'
   end
 
-  FormatParser.register_parser self, natures: :video, formats: FTYP_MAP.values
+  FormatParser.register_parser self, natures: :video, formats: FTYP_MAP.values, priority: 1
 end

data/lib/parsers/mp3_parser.rb

@@ -275,5 +275,5 @@ class FormatParser::MP3Parser
     end
   end
 
-  FormatParser.register_parser self, natures: :audio, formats: :mp3
+  FormatParser.register_parser self, natures: :audio, formats: :mp3, priority: 99
 end

data/lib/parsers/pdf_parser.rb

@@ -17,5 +17,5 @@ class FormatParser::PDFParser
     FormatParser::Document.new(format: :pdf)
   end
 
-  FormatParser.register_parser self, natures: :document, formats: :pdf
+  FormatParser.register_parser self, natures: :document, formats: :pdf, priority: 1
 end

data/lib/parsers/png_parser.rb

@@ -70,5 +70,6 @@ class FormatParser::PNGParser
     )
   end
 
-  FormatParser.register_parser self, natures: :image, formats: :png
+  # Give it priority 1 since priority 0 is reserved for JPEG, our most popular
+  FormatParser.register_parser self, natures: :image, formats: :png, priority: 1
 end

data/lib/parsers/zip_parser.rb

@@ -54,5 +54,5 @@ class FormatParser::ZIPParser
     end
   end
 
-  FormatParser.register_parser self, natures: [:archive, :document], formats: :zip
+  FormatParser.register_parser self, natures: [:archive, :document], formats: :zip, priority: 2
 end

data/spec/format_parser_spec.rb

@@ -34,6 +34,36 @@ describe FormatParser do
       end
     end
 
+    it 'triggers parsers in a certain order that corresponds to the parser priorities' do
+      file_contents = StringIO.new('a' * 4096)
+
+      parsers_called_order = []
+      expect_any_instance_of(FormatParser::PNGParser).to receive(:call) { |instance|
+        parsers_called_order << instance.class
+        nil
+      }
+      expect_any_instance_of(FormatParser::MP3Parser).to receive(:call) { |instance|
+        parsers_called_order << instance.class
+        nil
+      }
+      expect_any_instance_of(FormatParser::ZIPParser).to receive(:call) { |instance|
+        parsers_called_order << instance.class
+        nil
+      }
+
+      FormatParser.parse(file_contents)
+
+      png_parser_idx = parsers_called_order.index(FormatParser::PNGParser)
+      mp3_parser_idx = parsers_called_order.index(FormatParser::MP3Parser)
+      zip_parser_idx = parsers_called_order.index(FormatParser::ZIPParser)
+
+      # The PNG parser should have been applied first
+      expect(png_parser_idx).to be < zip_parser_idx
+      # ...and the ZIP parser second (MP3 is the most omnivorous since there
+      # is no clear header or footer in the file
+      expect(mp3_parser_idx).to be > zip_parser_idx
+    end
+
     it "returns either a valid result or a nil for all fuzzed inputs at seed #{RSpec.configuration.seed}" do
       r = Random.new(RSpec.configuration.seed)
       1024.times do
@@ -63,6 +93,14 @@ describe FormatParser do
       FormatParser.deregister_parser(exploit)
     end
 
+    it 'correctly detects a PNG as a PNG without falling back to another filetype' do
+      File.open(fixtures_dir + '/PNG/simulator_screenie.png', 'rb') do |file|
+        file_information = FormatParser.parse(file)
+        expect(file_information).not_to be_nil
+        expect(file_information.format).to eq(:png)
+      end
+    end
+
     describe 'when multiple results are requested' do
       let(:blob) { StringIO.new(Random.new.bytes(512 * 1024)) }
       let(:audio) { FormatParser::Audio.new(format: :aiff, num_audio_channels: 1) }

data/spec/remote_fetching_spec.rb

@@ -72,6 +72,13 @@ describe 'Fetching data from HTTP remotes' do
     end
   end
 
+  it 'correctly detects a PNG as a PNG without falling back to another filetype' do
+    remote_png_url = 'http://localhost:9399/PNG/simulator_screenie.png'
+    file_information = FormatParser.parse_http(remote_png_url)
+    expect(file_information).not_to be_nil
+    expect(file_information.format).to eq(:png)
+  end
+
   describe 'when parsing remote fixtures' do
     Dir.glob(fixtures_dir + '/**/*.*').sort.each do |fixture_path|
       filename = File.basename(fixture_path)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.14.1
+  version: 0.15.0
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-27 00:00:00.000000000 Z
+date: 2018-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks