format_parser 0.25.2 → 0.26.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 290b2369d2fe089202a76ab9d7b659f8de58fa44b9a40718130105ae7026036a
-  data.tar.gz: 53e983a8639cc42377ab50d7364b5099cf1f3308f8108f92b6194040546ea2e8
+  metadata.gz: 1074a8172a2830a11df0fb7874936c2b8abab8d74fd39985f9c9f7b72d5b348c
+  data.tar.gz: 5eafd2d610cd30bc85a056bd1c31331bcb49e8ce6b5b538cd13e280b138be7db
 SHA512:
-  metadata.gz: 5f58620ab165b1c47a8a18fe82081d48cb0b285b4f9638146c31d1bb2f839247b36750a7f85be5a9ea14b8db3fc2ca175b86ab4f8b29f87bd1ea10caed57746c
-  data.tar.gz: b9e87723a7cc1d5ecf04a23c28cbfc433c120337275aa51d76396a9a1371bb1b683b22c18ab4e84d8870a76048e26f7c5d7f90b4b5f9b868bf5a0dcb6771640c
+  metadata.gz: 5ce396a71fedd82b8041bcb6c833e559c7eef74886e73095eaf0b3d21e0c0d49b1620a83aba9796a8134dd5a7fc679156cd967cf82cba95b5941941be73d70c4
+  data.tar.gz: '0395e5a8fb35e860060e9c3b040b788aaad97eb5883f2d662b418156f1f3986bc4a26a814b9a8552ce7dcbd271da27e977cfb77a5e5b155ebd35db6a49a97719'

data/.travis.yml

@@ -2,8 +2,9 @@ rvm:
 - 2.2.10
 - 2.3.8
 - 2.4.9
-- 2.5.7
-- 2.6.5
+- 2.5.8
+- 2.6.6
+- 2.7.2
 - jruby
 sudo: false
 cache: bundler

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+## 0.26.0
+* Add support for M3U format files
+
+## 0.25.6
+* Fix FormatParser.parse (with `results: :first`) to be deterministic
+
+## 0.25.5
+* DPX: Fix DPXParser to support images without aspect ratio
+
+## 0.25.4
+* MP3: Fix MP3Parser to return nil for TIFF files
+* Add support to ruby 2.7
+
+## 0.25.3
+* MP3: Fix parser to not skip the first bytes if it's not an ID3 header
+
+## 0.25.2
+* Hotfix Moov parser
+
 ## 0.25.1
 * MOV: Fix error "negative length"
 * MOV: Fix reading dimensions in multi-track files

data/CONTRIBUTING.md

@@ -83,32 +83,59 @@ of software. Ideally, this file is going to be something you have produced yours
 and you are permitted to share under the MIT license provisions.
 
 When writing a parser, please try to ensure it returns a usable result as soon as possible,
-or no result as soon as possible (once you know the file is not fit for your specific parser).
+or `nil` as soon as possible (once you know the file is not fit for your specific parser).
 Bear in mind that we enforce read budgets per-parser, so you will not be allowed to perform
 too many reads, or perform reads which are too large.
 
-In order to create new parsers, it is recommended to make a well-named class with an instance method `call`.
+In order to create new parsers, make a well-named class with an instance method `call`,
+and to register a single instance of that class as the parser - so that only one object needs to be stored
+in memory when parsing multiple inputs. In that case your object must be **thread-safe and stateless** - this
+is really important since FormatParser is thread-safe and multiple parsing procedures may be in progress
+concurrently against the same parser object. You can also create a Proc if your parser is fairly trivial.
 
-`call` accepts the IO-ish object as an argument, parses data that it reads from it,
-and then returns the metadata for the file (if it could recover any) or `nil` if it couldn't. All files pass
-through all parsers by default, so if you are dealing with a file that is not "your" format - return `nil` from
-your method or `break` your Proc as early as possible. A blank `return` works fine too.
+If it will be difficult to have your parser thread-safe you can register your class itself as
+the parser and define the `self.call` method to parse using a fresh instance every time, allowing
+object-level state:
+
+```ruby
+class MyParser
+  def self.call(io)
+    new.call(io)
+  end
+
+  def call(io)
+    @state = ...
+  end
+```
+
+`call` accepts a single argument - an IO-ish object which is guaranteed to respond to the same methods as the
+ones defined in `IOConstraint` - that is, it is a strict subset of a standard Ruby IO object. *All reads from
+this IO object are guaranteed to be returned in binary encoding.* The IO will be at offset of 0 when your parsing
+proc receives it and there will be no concurrent calls to that object until your proc returns.
 
-The IO will at the minimum support the subset of the IO API defined in `IOConstraint`
+Your parsing procedure may read from this IO object, and should return either a `Result`-like object with
+the file metadata (if it could recover any) or `nil` if it couldn't. All files pass
+through all parsers by default, so if you are dealing with a file that is not "your" format - return `nil` from
+your method or `break` your Proc as early as possible. A blank `return` works fine too as it actually returns `nil`.
 
-Your parser has to be registered using `FormatParser.register_parser` with the information on the formats
-and file natures it provides.
+Your parser then needs to be registered using `FormatParser.register_parser` with the information on the formats
+and file natures it provides. This allows FormatParser to skip your parser if, say, the user only want to parse for
+`:image` nature files but your parser parses `:audio`.
 
-Down below you can find the most basic parser implementation:
+Down below you can find the most basic parser implementation which parses an imaginary `IMGA` file format:
 
 ```ruby
 MyParser = ->(io) {
-  # ... do some parsing with `io`
+  # ... Read the magic bytes from the start of IO - the IO is
+  # guaranteed to be fed to you at offset 0, start-of-file.
   magic_bytes = io.read(4)
+
   # breaking the block returns `nil` to the caller signaling "no match"
   break if magic_bytes != 'IMGA'
 
+  # Our file format stores the width and height as 2 32-bit unsigned integers
   parsed_witdh, parsed_height = io.read(8).unpack('VV')
+
   # ...and return the FileInformation::Image object with the metadata.
   FormatParser::Image.new(
     format: :imga,
@@ -135,8 +162,8 @@ class MyParser
     # ... do some parsing with `io`
     # The instance will be discarded after parsing, so using instance variables
     # is permitted - they are not shared between calls to `call`
-    @magic_bytes = io.read(4)
-    break if @magic_bytes != 'IMGA'
+    magic_bytes = io.read(4)
+    break if magic_bytes != 'IMGA'
     parsed_witdh, parsed_height = io.read(8).unpack('VV')
     FormatParser::Image.new(
       format: :imga,
@@ -145,23 +172,33 @@ class MyParser
     )
   end
 
-  FormatParser.register_parser self, natures: :image, formats: :bmp
+  # Note that we register an instance of the class, not the class. It is the
+  # instance that responds to `call()` and we can do this because our object
+  # is stateless.
+  FormatParser.register_parser new, natures: :image, formats: :bmp
 end
 ```
 
-### Calling convention for preparing parsers
+If your parser supports file types which have a known filename extension, you can add a method to it called `likely_match?`,
+add this method on the object you register itself. For example, for the ZIP parser we use:
+
+```ruby
+  def likely_match?(filename)
+    filename =~ /\.(zip|docx|keynote|numbers|pptx|xlsx)$/i
+  end
+```
 
-A parser that gets registered using `register_parser` must be either:
+If your parser matches the filename it is going to be applied *earlier*, saving time. Since most FormatParser users are
+likely to only want the first result of the parsing, the sooner your parser gets applied - the sooner you can return the result,
+avoiding unnecessary reads.
 
-1) An object that can be `call()`-ed itself, with an argument that conforms to `IOConstraint`
-2) An object that responds to `new` and returns something that can be `call()`-ed with with an argument that conforms to `IOConstraint`.
+### Calling convention for preparing parsers
 
-The second opton is recommended  for most cases.
+A parser that gets registered using `register_parser` must be an object that can be `call()`-ed, with an argument that conforms to `IOConstraint`
 
 FormatParser is made to be used in threaded environments, and if you use instance variables
-you need your parser to be isolated from it's siblings in other threads - therefore you can pass
-a Class on registration to have your parser instantiated for each `call()`, anew.
-
+you need your parser to be isolated from it's siblings in other threads - create a copy for one-off use inside
+your `call` method.
 
 ## Pull requests
 

data/README.md

@@ -32,6 +32,7 @@ and [dimensions,](https://github.com/sstephenson/dimensions) borrowing from them
 * DOCX, PPTX, XLSX
 * OGG
 * MPEG, MPG
+* M3U
 
 ...with [more](https://github.com/WeTransfer/format_parser/issues?q=is%3Aissue+is%3Aopen+label%3Aformats) on the way!
 
@@ -194,6 +195,9 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
   manipulated using the [https://github.com/recurser/exif-orientation-examples](exif-orientation-examples)
   script.
 
+### M3U
+- The M3U fixture files were created by one of the project maintainers
+
 ### .key
 - The `keynote_recognized_as_jpeg.key` file was created by the project maintainers
 

data/lib/format_parser.rb

@@ -19,6 +19,7 @@ module FormatParser
   require_relative 'io_constraint'
   require_relative 'care'
   require_relative 'active_storage/blob_analyzer'
+  require_relative 'text'
 
   # Define Measurometer in the internal namespace as well
   # so that we stay compatible for the applications that use it
@@ -36,7 +37,7 @@ module FormatParser
   # Register a parser object to be used to perform file format detection. Each parser FormatParser
   # provides out of the box registers itself using this method.
   #
-  # @param callable_or_responding_to_new[#call, #new] an object that either responds to #new or to #call
+  # @param callable_parser[#call] an object that responds to #call for parsing an IO
   # @param formats[Array<Symbol>] file formats that the parser provides
   # @param natures[Array<Symbol>] file natures that the parser provides
   # @param priority[Integer] whether the parser has to be applied first or later. Parsers that offer the safest
@@ -45,39 +46,41 @@ module FormatParser
   #   with a lower priority value will be applied first, and if a single result is requested, will also return
   #   first.
   # @return void
-  def self.register_parser(callable_or_responding_to_new, formats:, natures:, priority: LEAST_PRIORITY)
+  def self.register_parser(callable_parser, formats:, natures:, priority: LEAST_PRIORITY)
     parser_provided_formats = Array(formats)
     parser_provided_natures = Array(natures)
     PARSER_MUX.synchronize do
-      @parsers ||= Set.new
-      @parsers << callable_or_responding_to_new
+      # It can't be a Set because the method `parsers_for` depends on the order
+      # that the parsers were added.
+      @parsers ||= []
+      @parsers << callable_parser unless @parsers.include?(callable_parser)
       @parsers_per_nature ||= {}
       parser_provided_natures.each do |provided_nature|
         @parsers_per_nature[provided_nature] ||= Set.new
-        @parsers_per_nature[provided_nature] << callable_or_responding_to_new
+        @parsers_per_nature[provided_nature] << callable_parser
       end
       @parsers_per_format ||= {}
       parser_provided_formats.each do |provided_format|
         @parsers_per_format[provided_format] ||= Set.new
-        @parsers_per_format[provided_format] << callable_or_responding_to_new
+        @parsers_per_format[provided_format] << callable_parser
       end
       @parser_priorities ||= {}
-      @parser_priorities[callable_or_responding_to_new] = priority
+      @parser_priorities[callable_parser] = priority
     end
   end
 
   # Deregister a parser object (makes FormatParser forget this parser existed). Is mostly used in
   # tests, but can also be used to forcibly disable some formats completely.
   #
-  # @param callable_or_responding_to_new[#call, #new] an object that either responds to #new or to #call
+  # @param callable_parser[#==] an object that is identity-equal to any other registered parser
   # @return void
-  def self.deregister_parser(callable_or_responding_to_new)
+  def self.deregister_parser(callable_parser)
     # Used only in tests
     PARSER_MUX.synchronize do
-      (@parsers || []).delete(callable_or_responding_to_new)
-      (@parsers_per_nature || {}).values.map { |e| e.delete(callable_or_responding_to_new) }
-      (@parsers_per_format || {}).values.map { |e| e.delete(callable_or_responding_to_new) }
-      (@parser_priorities || {}).delete(callable_or_responding_to_new)
+      (@parsers || []).delete(callable_parser)
+      (@parsers_per_nature || {}).values.map { |e| e.delete(callable_parser) }
+      (@parsers_per_format || {}).values.map { |e| e.delete(callable_parser) }
+      (@parser_priorities || {}).delete(callable_parser)
     end
   end
 
@@ -255,7 +258,19 @@ module FormatParser
     # Order the parsers according to their priority value. The ones having a lower
     # value will sort higher and will be applied sooner
     parsers_in_order_of_priority = parsers.to_a.sort do |parser_a, parser_b|
-      @parser_priorities[parser_a] <=> @parser_priorities[parser_b]
+      if @parser_priorities[parser_a] != @parser_priorities[parser_b]
+        @parser_priorities[parser_a] <=> @parser_priorities[parser_b]
+      else
+        # Some parsers have the same priority and we want them to be always sorted
+        # in the same way, to not change the result of FormatParser.parse(results: :first).
+        # When this changes, it can generate flaky tests or event different
+        # results in different environments, which can be hard to understand why.
+        # There is also no guarantee in the order that the elements are added in
+        # @@parser_priorities
+        # So, to have always the same order, we sort by the order that the parsers
+        # were registered if the priorities are the same.
+        @parsers.index(parser_a) <=> @parsers.index(parser_b)
+      end
     end
 
     # If there is one parser that is more likely to match, place it first

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.25.2'
+  VERSION = '0.26.0'
 end

data/lib/parsers/dpx_parser.rb

@@ -35,18 +35,23 @@ class FormatParser::DPXParser
     w = dpx_structure.fetch(:image).fetch(:pixels_per_line)
     h = dpx_structure.fetch(:image).fetch(:lines_per_element)
 
+    display_w = w
+    display_h = h
+
     pixel_aspect_w = dpx_structure.fetch(:orientation).fetch(:horizontal_pixel_aspect)
     pixel_aspect_h = dpx_structure.fetch(:orientation).fetch(:vertical_pixel_aspect)
-    pixel_aspect = pixel_aspect_w / pixel_aspect_h.to_f
 
-    image_aspect = w / h.to_f * pixel_aspect
+    # Find display height and width based on aspect only if the file structure has pixel aspects
+    if pixel_aspect_h != 0 && pixel_aspect_w != 0
+      pixel_aspect = pixel_aspect_w / pixel_aspect_h.to_f
 
-    display_w = w
-    display_h = h
-    if image_aspect > 1
-      display_h = (display_w / image_aspect).round
-    else
-      display_w = (display_h * image_aspect).round
+      image_aspect = w / h.to_f * pixel_aspect
+
+      if image_aspect > 1
+        display_h = (display_w / image_aspect).round
+      else
+        display_w = (display_h * image_aspect).round
+      end
     end
 
     FormatParser::Image.new(

data/lib/parsers/m3u_parser.rb

@@ -0,0 +1,21 @@
+class FormatParser::M3UParser
+  include FormatParser::IOUtils
+
+  HEADER = '#EXTM3U'
+
+  def likely_match?(filename)
+    filename =~ /\.m3u8?$/i
+  end
+
+  def call(io)
+    io = FormatParser::IOConstraint.new(io)
+
+    header = safe_read(io, 7)
+    return unless HEADER.eql?(header)
+
+    FormatParser::Text.new(
+      format: :m3u
+    )
+  end
+  FormatParser.register_parser new, natures: :text, formats: :m3u
+end

data/lib/parsers/mp3_parser.rb

@@ -29,6 +29,10 @@ class FormatParser::MP3Parser
   ZIP_LOCAL_ENTRY_SIGNATURE = "PK\x03\x04\x14\x00".b
   PNG_HEADER_BYTES = [137, 80, 78, 71, 13, 10, 26, 10].pack('C*')
 
+  MAGIC_LE = [0x49, 0x49, 0x2A, 0x0].pack('C4')
+  MAGIC_BE = [0x4D, 0x4D, 0x0, 0x2A].pack('C4')
+  TIFF_HEADER_BYTES = [MAGIC_LE, MAGIC_BE]
+
   # Wraps the Tag object returned by ID3Tag in such
   # a way that a usable JSON representation gets
   # returned
@@ -68,11 +72,16 @@ class FormatParser::MP3Parser
     return if header.start_with?(ZIP_LOCAL_ENTRY_SIGNATURE)
     return if header.start_with?(PNG_HEADER_BYTES)
 
+    io.seek(0)
+    return if TIFF_HEADER_BYTES.include?(safe_read(io, 4))
+
     # Read all the ID3 tags (or at least attempt to)
     io.seek(0)
     id3v1 = ID3Extraction.attempt_id3_v1_extraction(io)
     tags = [id3v1, ID3Extraction.attempt_id3_v2_extraction(io)].compact
 
+    io.seek(0) if tags.empty?
+
     # Compute how many bytes are occupied by the actual MPEG frames
     ignore_bytes_at_tail = id3v1 ? 128 : 0
     ignore_bytes_at_head = io.pos

data/lib/parsers/tiff_parser.rb

@@ -4,6 +4,7 @@ class FormatParser::TIFFParser
 
   MAGIC_LE = [0x49, 0x49, 0x2A, 0x0].pack('C4')
   MAGIC_BE = [0x4D, 0x4D, 0x0, 0x2A].pack('C4')
+  HEADER_BYTES = [MAGIC_LE, MAGIC_BE]
 
   def likely_match?(filename)
     filename =~ /\.tiff?$/i
@@ -12,7 +13,7 @@ class FormatParser::TIFFParser
   def call(io)
     io = FormatParser::IOConstraint.new(io)
 
-    return unless [MAGIC_LE, MAGIC_BE].include?(safe_read(io, 4))
+    return unless HEADER_BYTES.include?(safe_read(io, 4))
     io.seek(io.pos + 2) # Skip over the offset of the IFD, EXIFR will re-read it anyway
     return if cr2?(io)
 

data/lib/text.rb

@@ -0,0 +1,18 @@
+module FormatParser
+  class Text
+    include FormatParser::AttributesJSON
+
+    NATURE = :text
+
+    attr_accessor :format
+
+    # Only permits assignments via defined accessors
+    def initialize(**attributes)
+      attributes.map { |(k, v)| public_send("#{k}=", v) }
+    end
+
+    def nature
+      NATURE
+    end
+  end
+end

data/spec/format_parser_spec.rb

@@ -173,6 +173,26 @@ describe FormatParser do
       prioritized_parsers = FormatParser.parsers_for([:archive, :document, :image, :audio], [:tif, :jpg, :zip, :docx, :mp3, :aiff], 'a-file.zip')
       expect(prioritized_parsers.first).to be_kind_of(FormatParser::ZIPParser)
     end
+
+    it 'sorts the parsers by priority and name' do
+      parsers = FormatParser.parsers_for(
+        [:audio, :image],
+        [:cr2, :dpx, :fdx, :flac, :gif, :jpg, :mov, :mp4, :m4a, :mp3, :mpg, :mpeg, :ogg, :png, :tif, :wav]
+      )
+
+      expect(parsers.map { |parser| parser.class.name }).to eq([
+        'FormatParser::GIFParser',
+        'Class',
+        'FormatParser::PNGParser',
+        'FormatParser::CR2Parser',
+        'FormatParser::DPXParser',
+        'FormatParser::FLACParser',
+        'FormatParser::MP3Parser',
+        'FormatParser::OggParser',
+        'FormatParser::TIFFParser',
+        'FormatParser::WAVParser'
+      ])
+    end
   end
 
   describe '.register_parser and .deregister_parser' do

data/spec/parsers/m3u_parser_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+describe FormatParser::M3UParser do
+  let(:parsed_m3u) do
+    subject.call(
+      File.open(
+        Pathname.new(fixtures_dir).join('M3U').join(m3u_file),
+        'rb'
+      )
+    )
+  end
+
+  describe 'an m3u file with missing header' do
+    let(:m3u_file) { 'plain_text.m3u' }
+
+    it 'does not parse the file successfully' do
+      expect(parsed_m3u).to be_nil
+    end
+  end
+
+  describe 'an m3u file with valid header' do
+    let(:m3u_file) { 'sample.m3u' }
+
+    it 'parses the file successfully' do
+      expect(parsed_m3u).not_to be_nil
+      expect(parsed_m3u.nature).to eq(:text)
+      expect(parsed_m3u.format).to eq(:m3u)
+    end
+  end
+
+  describe 'an m3u8 file with valid header' do
+    let(:m3u_file) { 'sample.m3u8' }
+
+    it 'parses the file successfully' do
+      expect(parsed_m3u).not_to be_nil
+      expect(parsed_m3u.nature).to eq(:text)
+      expect(parsed_m3u.format).to eq(:m3u)
+    end
+  end
+end

data/spec/parsers/mp3_parser_spec.rb

@@ -166,6 +166,18 @@ describe FormatParser::MP3Parser do
     expect(parsed.artist). to eq('wetransfer')
   end
 
+  it 'does not skip the first bytes if it is not a id3 tag header' do
+    fpath = fixtures_dir + '/MP3/no_id3_tags.mp3'
+
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed).not_to be_nil
+
+    expect(parsed.nature).to eq(:audio)
+    expect(parsed.format).to eq(:mp3)
+    expect(parsed.audio_sample_rate_hz).to eq(44100)
+  end
+
   describe '#as_json' do
     it 'converts all hash keys to string when stringify_keys: true' do
       fpath = fixtures_dir + '/MP3/Cassy.mp3'
@@ -193,4 +205,12 @@ describe FormatParser::MP3Parser do
       ).to eq([ID3Tag::Tag])
     end
   end
+
+  it 'does not recognize TIFF files as MP3' do
+    fpath = fixtures_dir + '/TIFF/test2.tif'
+
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed).to be_nil
+  end
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.25.2
+  version: 0.26.0
 platform: ruby
 authors:
 - Noah Berman
 - Julik Tarkhanov
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-10-05 00:00:00.000000000 Z
+date: 2021-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -219,6 +219,7 @@ files:
 - lib/parsers/flac_parser.rb
 - lib/parsers/gif_parser.rb
 - lib/parsers/jpeg_parser.rb
+- lib/parsers/m3u_parser.rb
 - lib/parsers/moov_parser.rb
 - lib/parsers/moov_parser/decoder.rb
 - lib/parsers/mp3_parser.rb
@@ -236,6 +237,7 @@ files:
 - lib/read_limiter.rb
 - lib/read_limits_config.rb
 - lib/remote_io.rb
+- lib/text.rb
 - lib/video.rb
 - spec/active_storage/blob_io_spec.rb
 - spec/active_storage/rails_app_spec.rb
@@ -257,6 +259,7 @@ files:
 - spec/parsers/flac_parser_spec.rb
 - spec/parsers/gif_parser_spec.rb
 - spec/parsers/jpeg_parser_spec.rb
+- spec/parsers/m3u_parser_spec.rb
 - spec/parsers/moov_parser_spec.rb
 - spec/parsers/mp3_parser_spec.rb
 - spec/parsers/mpeg_parser_spec.rb
@@ -277,7 +280,7 @@ licenses:
 - MIT (Hippocratic)
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -292,8 +295,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.0.3
+signing_key:
 specification_version: 4
 summary: A library for efficient parsing of file metadata
 test_files: []