format_parser 0.15.1 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b54b8e7c496c5d1e732503678bc762993bf53e9a
-  data.tar.gz: ca7ce92f31f08a049b7475746e2017e6662b331a
+  metadata.gz: e49c10ea63c475b9bc61781ea8e76f42ef9e4307
+  data.tar.gz: c625105933c8654f5aac93bae09476e32917655b
 SHA512:
-  metadata.gz: d3dcc4792c6097dc701beb1b124ae17ad345305b4e7e4ac46a132646ff2aefea13cdc350087f8582b9b2653b75181c6c00ffb12d6a5180716538a732cfe60c46
-  data.tar.gz: 898ea93a39709415452b5256fdcee7acb69d9a1decc14c50b52695990ae399848c3f984787ab7374efb9b04b3bb758f0487a4b9da67c7fa83396ab5a5c114218
+  metadata.gz: cf72d2b021f6fc407b29cbc5b336fd71e022f6a40b67a79646ea58e605c33d34b44404339e7f15cf1c3f63729712a784ac75296fde54525f51a9c6276069619e
+  data.tar.gz: c68eadacfae8062e6fb1532d3e6976c349f1d3f7a950cafb7ea0bab714a125e81d3d32bccb154ed96c2118c142f984637ed17c6a53d1ee755778836ad008a7a0

data/.travis.yml

@@ -1,9 +1,9 @@
 rvm:
 - 2.2.0
-- 2.3.0
-- 2.4.2
-- 2.5.0
-- jruby-9.0
+- 2.4.6
+- 2.5.5
+- 2.6.3
+- jruby
 sudo: false
 cache: bundler
 script:

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.16.0
+* Add `filename_hint` keyword argument to `FormatParser.parse`. This can hint the library to apply
+  the parser that will likely match for this filename first, and the other parsers later. This helps
+  avoiding extra work when parsing less-popular file formats, and can be optionally used if the caller
+  knows the filename of the original file. Note that the filename is only that: a **hint,** it helps
+  apply parsers more efficiently but does not specify the actual format of the file that is going to
+  be detected.
+
+## 0.15.1
+* Relax the "ks" dependency version since we do not need the constraint to be so strict
+
 ## 0.15.0
 * Allow setting `:priority` when registering a parser, to make sure certain parsers are applied earlier - depending
   on detection confidence and file format popularity at WT.

data/lib/format_parser.rb

@@ -86,18 +86,22 @@ module FormatParser
   # @param kwargs the keyword arguments to be delegated to `.parse`
   # @see {.parse}
   def self.parse_http(url, **kwargs)
+    # Do not extract the filename, since the URL
+    # can really be "anything". But if the caller
+    # provides filename_hint it will be carried over
     parse(RemoteIO.new(url), **kwargs)
   end
 
   # Parses the file at the given `path` and returns the results as if it were any IO
   # given to `.parse`. The accepted keyword arguments are the same as the ones for `parse`.
+  # The file path will be used to provide the `filename_hint` to `.parse()`.
   #
   # @param path[String] the path to the file to parse on the local filesystem
   # @param kwargs the keyword arguments to be delegated to `.parse`
   # @see {.parse}
   def self.parse_file_at(path, **kwargs)
     File.open(path, 'rb') do |io|
-      parse(io, **kwargs)
+      parse(io, filename_hint: File.basename(path), **kwargs)
     end
   end
 
@@ -116,9 +120,13 @@ module FormatParser
   #   When using `:first` parsing will stop at the first successful match and other parsers won't run.
   # @param limits_config[ReadLimitsConfig] the configuration object for various read/cache limits. The default
   #   one should be good for most cases.
+  # @param filename_hint[String?] the filename. If provided, the first parser applied will be the
+  #   one that responds `true` to `likely_match?` with that filename as an argument. This way
+  #   we can optimize the order of application of parsers and start with the one that is more likely
+  #   to match.
   # @return [Array<Result>, Result, nil] either an Array of results, a single parsing result or `nil`if
   #   no useful metadata could be recovered from the file
-  def self.parse(io, natures: @parsers_per_nature.keys, formats: @parsers_per_format.keys, results: :first, limits_config: default_limits_config)
+  def self.parse(io, natures: @parsers_per_nature.keys, formats: @parsers_per_format.keys, results: :first, limits_config: default_limits_config, filename_hint: nil)
     # Limit the number of cached _pages_ we may fetch. This allows us to limit the number
     # of page faults (page cache misses) a parser may incur
     read_limiter_under_cache = FormatParser::ReadLimiter.new(io, max_reads: limits_config.max_pagefaults_per_parser)
@@ -140,7 +148,7 @@ module FormatParser
     # Always instantiate parsers fresh for each input, since they might
     # contain instance variables which otherwise would have to be reset
     # between invocations, and would complicate threading situations
-    parsers = parsers_for(natures, formats)
+    parsers = parsers_for(natures, formats, filename_hint)
 
     # Limit how many operations the parser can perform
     limited_io = ReadLimiter.new(
@@ -225,9 +233,11 @@ module FormatParser
   #
   # @param desired_natures[Array] which natures should be considered (like `[:image, :archive]`)
   # @param desired_formats[Array] which formats should be considered (like `[:tif, :jpg]`)
+  # @param filename_hint[String?] the filename hint for the file. If provided,
+  #     the parser that likely matches this filename will be applied first.
   # @return [Array<#call>] an array of callable parsers
   # @raise ArgumentError when there are no parsers satisfying the constraint
-  def self.parsers_for(desired_natures, desired_formats)
+  def self.parsers_for(desired_natures, desired_formats, filename_hint = nil)
     assemble_parser_set = ->(hash_of_sets, keys_of_interest) {
       hash_of_sets.values_at(*keys_of_interest).compact.inject(&:+) || Set.new
     }
@@ -246,6 +256,12 @@ module FormatParser
       @parser_priorities[parser_factory_a] <=> @parser_priorities[parser_factory_b]
     end
 
+    # If there is one parser that is more likely to match, place it first
+    if first_match = factories_in_order_of_priority.find { |f| f.respond_to?(:likely_match?) && f.likely_match?(filename_hint) }
+      factories_in_order_of_priority.delete(first_match)
+      factories_in_order_of_priority.unshift(first_match)
+    end
+
     factories_in_order_of_priority.map { |callable_or_class| instantiate_parser(callable_or_class) }
   end
 

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.15.1'
+  VERSION = '0.16.0'
 end

data/lib/parsers/aiff_parser.rb

@@ -18,6 +18,10 @@ class FormatParser::AIFFParser
     'ANNO',
   ]
 
+  def self.likely_match?(filename)
+    filename =~ /\.aiff?$/i
+  end
+
   def call(io)
     io = FormatParser::IOConstraint.new(io)
     form_chunk_type, chunk_size = safe_read(io, 8).unpack('a4N')

data/lib/parsers/bmp_parser.rb

@@ -6,6 +6,10 @@ class FormatParser::BMPParser
   VALID_BMP = 'BM'
   PERMISSIBLE_PIXEL_ARRAY_LOCATIONS = 40..512
 
+  def self.likely_match?(filename)
+    filename =~ /\.bmp$/i
+  end
+
   def call(io)
     io = FormatParser::IOConstraint.new(io)
 

data/lib/parsers/cr2_parser.rb

@@ -7,6 +7,10 @@ class FormatParser::CR2Parser
   TIFF_HEADER = [0x49, 0x49, 0x2a, 0x00]
   CR2_HEADER  = [0x43, 0x52, 0x02, 0x00]
 
+  def self.likely_match?(filename)
+    filename =~ /\.cr2$/i
+  end
+
   def call(io)
     io = FormatParser::IOConstraint.new(io)
 

data/lib/parsers/dpx_parser.rb

@@ -19,6 +19,10 @@ class FormatParser::DPXParser
 
   private_constant :ByteOrderHintIO
 
+  def self.likely_match?(filename)
+    filename =~ /\.dpx$/i
+  end
+
   def call(io)
     io = FormatParser::IOConstraint.new(io)
     magic = safe_read(io, 4)

data/lib/parsers/fdx_parser.rb

@@ -1,6 +1,10 @@
 class FormatParser::FDXParser
   include FormatParser::IOUtils
 
+  def self.likely_match?(filename)
+    filename =~ /\.fdx$/i
+  end
+
   def call(io)
     return unless xml_check(io)
     file_and_document_type = safe_read(io, 100)

data/lib/parsers/flac_parser.rb

@@ -5,8 +5,8 @@ class FormatParser::FLACParser
   MAGIC_BYTE_STRING = 'fLaC'
   BLOCK_HEADER_BYTES = 4
 
-  def bytestring_to_int(s)
-    s.unpack('B*')[0].to_i(2)
+  def self.likely_match?(filename)
+    filename =~ /\.flac$/i
   end
 
   def call(io)
@@ -71,5 +71,9 @@ class FormatParser::FLACParser
     )
   end
 
+  def bytestring_to_int(s)
+    s.unpack('B*')[0].to_i(2)
+  end
+
   FormatParser.register_parser self, natures: :audio, formats: :flac
 end

data/lib/parsers/gif_parser.rb

@@ -4,6 +4,10 @@ class FormatParser::GIFParser
   HEADERS = ['GIF87a', 'GIF89a'].map(&:b)
   NETSCAPE_AND_AUTHENTICATION_CODE = 'NETSCAPE2.0'
 
+  def self.likely_match?(filename)
+    filename =~ /\.gif$/i
+  end
+
   def call(io)
     io = FormatParser::IOConstraint.new(io)
     header = safe_read(io, 6)

data/lib/parsers/jpeg_parser.rb

@@ -13,6 +13,10 @@ class FormatParser::JPEGParser
   EXIF_MAGIC_STRING = "Exif\0\0".b
   MUST_FIND_NEXT_MARKER_WITHIN_BYTES = 1024
 
+  def self.likely_match?(filename)
+    filename =~ /\.jpe?g$/i
+  end
+
   def call(io)
     @buf = FormatParser::IOConstraint.new(io)
     @width             = nil

data/lib/parsers/moov_parser.rb

@@ -11,6 +11,10 @@ class FormatParser::MOOVParser
     'm4a ' => :m4a,
   }
 
+  def self.likely_match?(filename)
+    filename =~ /\.(mov|m4a|ma4|mp4|aac)$/i
+  end
+
   def call(io)
     return unless matches_moov_definition?(io)
 

data/lib/parsers/mp3_parser.rb

@@ -53,6 +53,10 @@ class FormatParser::MP3Parser
     end
   end
 
+  def self.likely_match?(filename)
+    filename =~ /\.mp3$/i
+  end
+
   def call(raw_io)
     io = FormatParser::IOConstraint.new(raw_io)
 

data/lib/parsers/ogg_parser.rb

@@ -6,6 +6,10 @@ class FormatParser::OggParser
   # Maximum size of an Ogg page
   MAX_POSSIBLE_PAGE_SIZE = 65307
 
+  def self.likely_match?(filename)
+    filename =~ /\.ogg$/i
+  end
+
   def call(io)
     # The format consists of chunks of data each called an "Ogg page". Each page
     # begins with the characters, "OggS", to identify the file as Ogg format.

data/lib/parsers/pdf_parser.rb

@@ -9,6 +9,10 @@ class FormatParser::PDFParser
   #
   PDF_MARKER = /%PDF-1\.[0-8]{1}/
 
+  def self.likely_match?(filename)
+    filename =~ /\.(pdf|ai)$/i
+  end
+
   def call(io)
     io = FormatParser::IOConstraint.new(io)
 

data/lib/parsers/png_parser.rb

@@ -15,8 +15,8 @@ class FormatParser::PNGParser
     6 => true,
   }
 
-  def chunk_length_and_type(io)
-    safe_read(io, 8).unpack('Na4')
+  def self.likely_match?(filename)
+    filename =~ /\.png$/i
   end
 
   def call(io)
@@ -70,6 +70,10 @@ class FormatParser::PNGParser
     )
   end
 
+  def chunk_length_and_type(io)
+    safe_read(io, 8).unpack('Na4')
+  end
+
   # Give it priority 1 since priority 0 is reserved for JPEG, our most popular
   FormatParser.register_parser self, natures: :image, formats: :png, priority: 1
 end

data/lib/parsers/psd_parser.rb

@@ -3,6 +3,10 @@ class FormatParser::PSDParser
 
   PSD_HEADER = [0x38, 0x42, 0x50, 0x53]
 
+  def self.likely_match?(filename)
+    filename =~ /\.psd$/i # Maybe also PSB at some point
+  end
+
   def call(io)
     io = FormatParser::IOConstraint.new(io)
     magic_bytes = safe_read(io, 4).unpack('C4')

data/lib/parsers/tiff_parser.rb

@@ -5,6 +5,10 @@ class FormatParser::TIFFParser
   MAGIC_LE = [0x49, 0x49, 0x2A, 0x0].pack('C4')
   MAGIC_BE = [0x4D, 0x4D, 0x0, 0x2A].pack('C4')
 
+  def self.likely_match?(filename)
+    filename =~ /\.tiff?$/i
+  end
+
   def call(io)
     io = FormatParser::IOConstraint.new(io)
 

data/lib/parsers/wav_parser.rb

@@ -1,6 +1,10 @@
 class FormatParser::WAVParser
   include FormatParser::IOUtils
 
+  def self.likely_match?(filename)
+    filename =~ /\.wav$/i
+  end
+
   def call(io)
     # Read the RIFF header. Chunk descriptor should be RIFF, the size should
     # contain the size of the entire file in bytes minus 8 bytes for the

data/lib/parsers/zip_parser.rb

@@ -5,6 +5,10 @@ class FormatParser::ZIPParser
   include OfficeFormats
   include FormatParser::IOUtils
 
+  def self.likely_match?(filename)
+    filename =~ /\.(zip|docx|keynote|numbers|pptx|xlsx)$/i
+  end
+
   def call(io)
     io = FormatParser::IOConstraint.new(io)
     safe_read(io, 1) # Ensure the file is not empty

data/spec/format_parser_spec.rb

@@ -139,7 +139,7 @@ describe FormatParser do
 
     it 'passes keyword arguments to parse()' do
       path = fixtures_dir + '/WAV/c_M1F1-Alaw-AFsp.wav'
-      expect(FormatParser).to receive(:parse).with(an_instance_of(File), foo: :bar)
+      expect(FormatParser).to receive(:parse).with(an_instance_of(File), filename_hint: 'c_M1F1-Alaw-AFsp.wav', foo: :bar)
       FormatParser.parse_file_at(path, foo: :bar)
     end
   end
@@ -165,6 +165,14 @@ describe FormatParser do
       image_parsers = FormatParser.parsers_for([:image], [:tif, :jpg, :aiff, :mp3])
       expect(image_parsers.length).to eq(2)
     end
+
+    it 'returns an array with the ZIPParser first if the filename_hint is for a ZIP file' do
+      prioritized_parsers = FormatParser.parsers_for([:archive, :document, :image, :audio], [:tif, :jpg, :zip, :docx, :mp3, :aiff], nil)
+      expect(prioritized_parsers.first).not_to be_kind_of(FormatParser::ZIPParser)
+
+      prioritized_parsers = FormatParser.parsers_for([:archive, :document, :image, :audio], [:tif, :jpg, :zip, :docx, :mp3, :aiff], 'a-file.zip')
+      expect(prioritized_parsers.first).to be_kind_of(FormatParser::ZIPParser)
+    end
   end
 
   describe '.register_parser and .deregister_parser' do

data/spec/parsers/zip_parser_spec.rb

@@ -1,6 +1,11 @@
 require 'spec_helper'
 
 describe FormatParser::ZIPParser do
+  it 'provides filename hints' do
+    expect(FormatParser::ZIPParser).to be_likely_match('file.zip')
+    expect(FormatParser::ZIPParser).not_to be_likely_match('file.tif')
+  end
+
   it 'parses a ZIP archive with Zip64 extra fields (due to the number of files)' do
     fixture_path = fixtures_dir + '/ZIP/arch_many_entries.zip'
     fi_io = File.open(fixture_path, 'rb')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.15.1
+  version: 0.16.0
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-29 00:00:00.000000000 Z
+date: 2019-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks