format_parser 0.23.1 → 0.25.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f76db376646606abe6a7ccaa6f0a14efdc997ecd7fe29aff0ab3d8172857649f
-  data.tar.gz: 07c142e7ce6aaa518285d425eb95961d18e053a6b15456d4cb569cdc70a79069
+  metadata.gz: 87adbfef15c2281ab6a13f151b857409f0ffad0ecc5270d9d0bbc5cebe207cdb
+  data.tar.gz: 332e3c4efd4ae01b3cf47c669debba7cc1aee5c264bef503720f632f6d801054
 SHA512:
-  metadata.gz: 3e92625dbe822d3423a084174fd7bc9d4a296effe26182a91743b36995f8470edc4fa56a3a0a10ce095d4998da5dee9620b983c1f980e0418a3b4574244e848c
-  data.tar.gz: b89df24f7b6638b3cd42b284a1792475a094e947bd995e5ccdd0c554014075f5d167a7c04fbc08c681483b0c82aa2e59ae4e4af62f99d6b158fb4b3f1096d80b
+  metadata.gz: 231d768d4b69b2c2f29bcb861888a7fbb0f4242eec5c9313d6428c9053b4fb4e7d20b8615d731a695c278ac59d3bf07b4977d217892aa1fce8fa7adc9d415efa
+  data.tar.gz: 732fae92e71f6a25fa98b7d88fa70b82404f00c234a6debd641e20ad8ffd9e45c78270f20adb8ad5593cd815e128a2a409ec639351c985b79763d4bfd821fec9

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+## 0.25.1
+* MOV: Fix error "negative length"
+* MOV: Fix reading dimensions in multi-track files
+* MP3: Fix parse of the Xing header to not raise errors
+
+## 0.25.0
+* MP3: add suport to id3 v2.4.x
+* JPEG: Update gem exifr to 1.3.8 to fix a bug
+
+## 0.24.2
+* Update gem id3tag to 0.14.0 to fix MP3 issues
+
+## 0.24.1
+* Fix MP3 frames reading to jump correctly to the next bytes
+
+## 0.24.0
+* The TIFF parser will now return :arw as format for Sony ARW files insted of :tif so that the caller can decide whether it
+  wants to deal with RAW processing or not
+
 ## 0.23.1
 * Updated gem exifr to fix problems related to jpeg files from Olympos microscopes, which often have bad thumbnail data
 

data/format_parser.gemspec

@@ -31,8 +31,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'ks', '~> 0.0'
-  spec.add_dependency 'exifr', '~> 1', '>= 1.3.7'
-  spec.add_dependency 'id3tag', '~> 0.13'
+  spec.add_dependency 'exifr', '~> 1', '>= 1.3.8'
+  spec.add_dependency 'id3tag', '~> 0.14'
   spec.add_dependency 'faraday', '~> 0.13'
   spec.add_dependency 'measurometer', '~> 1'
 

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.23.1'
+  VERSION = '0.25.1'
 end

data/lib/parsers/moov_parser.rb

@@ -38,14 +38,8 @@ class FormatParser::MOOVParser
     ftyp_atom = decoder.find_first_atom_by_path(atom_tree, 'ftyp')
     file_type = ftyp_atom.field_value(:major_brand)
 
-    width = nil
-    height = nil
-
     # Try to find the width and height in the tkhd
-    if tkhd = decoder.find_first_atom_by_path(atom_tree, 'moov', 'trak', 'tkhd')
-      width = tkhd.field_value(:track_width).first
-      height = tkhd.field_value(:track_height).first
-    end
+    width, height = parse_dimensions(decoder, atom_tree)
 
     # Try to find the "topmost" duration (respecting edits)
     if mdhd = decoder.find_first_atom_by_path(atom_tree, 'moov', 'mvhd')
@@ -78,6 +72,31 @@ class FormatParser::MOOVParser
     FTYP_MAP.fetch(file_type.downcase, :mov)
   end
 
+  # The dimensions are located in tkhd atom, but in some files it is necessary
+  # to get it below the video track, because it can have other tracks such as
+  # audio which does not have the dimensions.
+  # More details in https://developer.apple.com/library/archive/documentation/QuickTime/QTFF/QTFFChap2/qtff2.html#//apple_ref/doc/uid/TP40000939-CH204-DontLinkElementID_147
+  #
+  # Returns [width, height] if the dimension is found
+  # Returns [nil, nil] if the dimension is not found
+  def parse_dimensions(decoder, atom_tree)
+    video_trak_atom = decoder.find_video_trak_atom(atom_tree)
+
+    tkhd = begin
+      if video_trak_atom
+        decoder.find_first_atom_by_path([video_trak_atom], 'trak', 'tkhd')
+      else
+        decoder.find_first_atom_by_path(atom_tree, 'moov', 'trak', 'tkhd')
+      end
+    end
+
+    if tkhd
+      [tkhd.field_value(:track_width).first, tkhd.field_value(:track_height).first]
+    else
+      [nil, nil]
+    end
+  end
+
   # An MPEG4/MOV/M4A will start with the "ftyp" atom. The atom must have a length
   # of at least 8 (to accomodate the atom size and the atom type itself) plus the major
   # and minor version fields. If we cannot find it we can be certain this is not our file.

data/lib/parsers/moov_parser/decoder.rb

@@ -1,6 +1,7 @@
 # Handles decoding of MOV/MPEG4 atoms/boxes in a stream. Will recursively
 # read atoms and parse their data fields if applicable. Also contains
 # a few utility functions for finding atoms in a list etc.
+# To know more about Atoms: https://developer.apple.com/library/archive/documentation/QuickTime/QTFF/QTFFChap2/qtff2.html
 class FormatParser::MOOVParser::Decoder
   include FormatParser::IOUtils
 
@@ -47,6 +48,34 @@ class FormatParser::MOOVParser::Decoder
     find_first_atom_by_path(requisite.children || [], *atom_types)
   end
 
+  def find_atoms_by_path(atoms, atom_types)
+    type_to_find = atom_types.shift
+    requisites = atoms.select { |e| e.atom_type == type_to_find }
+
+    # Return if we found our match
+    return requisites if atom_types.empty?
+
+    # Return nil if we didn't find the match at this nesting level
+    return unless requisites
+
+    # ...otherwise drill further down
+    find_atoms_by_path(requisites.flat_map(&:children).compact || [], atom_types)
+  end
+
+  # A file can have multiple tracks. To identify the type it is necessary to check
+  # the fields `omponent_subtype` in hdlr atom under the trak atom
+  # More details in https://developer.apple.com/library/archive/documentation/QuickTime/QTFF/QTFFChap2/qtff2.html#//apple_ref/doc/uid/TP40000939-CH204-DontLinkElementID_147
+  def find_video_trak_atom(atoms)
+    trak_atoms = find_atoms_by_path(atoms, ['moov', 'trak'])
+
+    return [] if trak_atoms.empty?
+
+    trak_atoms.find do |trak_atom|
+      hdlr_atom = find_first_atom_by_path([trak_atom], 'trak', 'mdia', 'hdlr')
+      hdlr_atom.atom_fields[:component_type] == 'mhlr' && hdlr_atom.atom_fields[:component_subtype] == 'vide'
+    end
+  end
+
   def parse_ftyp_atom(io, atom_size)
     # Subtract 8 for the atom_size+atom_type,
     # and 8 once more for the major_brand and minor_version. The remaining
@@ -194,6 +223,8 @@ class FormatParser::MOOVParser::Decoder
   end
 
   def parse_meta_atom(io, atom_size)
+    return if atom_size == 0 # this atom can be empty
+
     parse_hdlr_atom(io, atom_size)
   end
 

data/lib/parsers/mp3_parser.rb

@@ -20,13 +20,14 @@ class FormatParser::MP3Parser
 
   # We limit the number of MPEG frames we scan
   # to obtain our duration estimation
-  MAX_FRAMES_TO_SCAN = 128
+  MAX_FRAMES_TO_SCAN = 500
 
   # Default frame size for mp3
   SAMPLES_PER_FRAME = 1152
 
   # For some edge cases
   ZIP_LOCAL_ENTRY_SIGNATURE = "PK\x03\x04\x14\x00".b
+  PNG_HEADER_BYTES = [137, 80, 78, 71, 13, 10, 26, 10].pack('C*')
 
   # Wraps the Tag object returned by ID3Tag in such
   # a way that a usable JSON representation gets
@@ -60,8 +61,12 @@ class FormatParser::MP3Parser
     # To avoid having that happen, we check for the PKZIP signature -
     # local entry header signature - at the very start of the file.
     # If the file is too small safe_read will fail too and the parser
-    # will terminate here.
-    return if safe_read(io, 6) == ZIP_LOCAL_ENTRY_SIGNATURE
+    # will terminate here. Same with PNGs. In the future
+    # we should implement "confidence" for MP3 as of all our formats
+    # it is by far the most lax.
+    header = safe_read(io, 8)
+    return if header.start_with?(ZIP_LOCAL_ENTRY_SIGNATURE)
+    return if header.start_with?(PNG_HEADER_BYTES)
 
     # Read all the ID3 tags (or at least attempt to)
     io.seek(0)
@@ -131,27 +136,28 @@ class FormatParser::MP3Parser
   # if you have a minute. https://pypi.python.org/pypi/tinytag
   def parse_mpeg_frames(io)
     mpeg_frames = []
+    bytes_to_read = 4
 
     MAX_FRAMES_TO_SCAN.times do |frame_i|
       # Read through until we can latch onto the 11 sync bits. Read in 4-byte
       # increments to save on read() calls
-      data = io.read(4)
+      data = io.read(bytes_to_read)
 
       # If we are at EOF - stop iterating
-      break unless data && data.bytesize == 4
+      break unless data && data.bytesize == bytes_to_read
 
       # Look for the sync pattern. It can be either the last byte being 0xFF,
       # or any of the 2 bytes in sequence being 0xFF and > 0xF0.
       four_bytes = data.unpack('C4')
       seek_jmp = sync_bytes_offset_in_4_byte_seq(four_bytes)
       if seek_jmp > 0
-        io.seek(io.pos + seek_jmp)
+        io.seek(io.pos - bytes_to_read + seek_jmp)
         next
       end
 
       # Once we are past that stage we have latched onto a sync frame header
       sync, conf, bitrate_freq, rest = four_bytes
-      frame_detail = parse_mpeg_frame_header(io.pos - 4, sync, conf, bitrate_freq, rest)
+      frame_detail = parse_mpeg_frame_header(io.pos - bytes_to_read, sync, conf, bitrate_freq, rest)
       mpeg_frames << frame_detail
 
       # There might be a xing header in the first frame that contains
@@ -166,7 +172,7 @@ class FormatParser::MP3Parser
         end
       end
       if frame_detail.frame_length > 1 # jump over current frame body
-        io.seek(io.pos + frame_detail.frame_length - 4)
+        io.seek(io.pos + frame_detail.frame_length - bytes_to_read)
       end
     end
     [nil, mpeg_frames]
@@ -243,16 +249,16 @@ class FormatParser::MP3Parser
     io.seek(xing_offset + 4) # Include the length of "Xing" itself
 
     # https://www.codeproject.com/Articles/8295/MPEG-Audio-Frame-Header#XINGHeader
-    header_flags, _ = io.read(4).unpack('s>s>')
+    header_flags, _ = io.read(4).unpack('i>')
     frames = byte_count = toc = vbr_scale = nil
 
-    frames = io.read(4).unpack('N1').first if header_flags & 1 # FRAMES FLAG
+    frames = io.read(4).unpack('N1').first if header_flags & 1 != 0   # FRAMES FLAG
 
-    byte_count = io.read(4).unpack('N1').first if header_flags & 2 # BYTES FLAG
+    byte_count = io.read(4).unpack('N1').first if header_flags & 2 != 0   # BYTES FLAG
 
-    toc = io.read(100).unpack('C100') if header_flags & 4 # TOC FLAG
+    toc = io.read(100).unpack('C100') if header_flags & 4 != 0   # TOC FLAG
 
-    vbr_scale = io.read(4).unpack('N1').first if header_flags & 8 # VBR SCALE FLAG
+    vbr_scale = io.read(4).unpack('N1').first if header_flags & 8 != 0   # VBR SCALE FLAG
 
     VBRHeader.new(frames: frames, byte_count: byte_count, toc_entries: toc, vbr_scale: vbr_scale)
   end

data/lib/parsers/mp3_parser/id3_extraction.rb

@@ -1,6 +1,8 @@
 module FormatParser::MP3Parser::ID3Extraction
   ID3V1_TAG_SIZE_BYTES = 128
-  ID3V2_TAG_VERSIONS = ["\x43\x00".b, "\x03\x00".b, "\x02\x00".b]
+  # it supports 2.4.x, 2.3.x, 2.2.x which are supported by the gem id3tag
+  # see https://id3.org/Developer%20Information for more details of each version
+  ID3V2_MINOR_TAG_VERSIONS = [2, 3, 4]
   MAX_SIZE_FOR_ID3V2 = 1 * 1024 * 1024
 
   extend FormatParser::IOUtils
@@ -22,7 +24,7 @@ module FormatParser::MP3Parser::ID3Extraction
     io.seek(0) # Only support header ID3v2
     header = parse_id3_v2_header(io)
     return unless header[:tag] == 'ID3' && header[:size] > 0
-    return unless ID3V2_TAG_VERSIONS.include?(header[:version])
+    return unless ID3V2_MINOR_TAG_VERSIONS.include?(header[:version].unpack('C').first)
 
     id3_tag_size = io.pos + header[:size]
 

data/lib/parsers/tiff_parser.rb

@@ -26,7 +26,7 @@ class FormatParser::TIFFParser
     h = exif_data.height || exif_data.pixel_y_dimension
 
     FormatParser::Image.new(
-      format: :tif,
+      format: arw?(exif_data) ? :arw : :tif, # Specify format as arw for Sony ARW format images, else tif
       width_px: w,
       height_px: h,
       display_width_px: exif_data.rotated? ? h : w,
@@ -43,5 +43,11 @@ class FormatParser::TIFFParser
     safe_read(io, 2) == 'CR'
   end
 
+  # Similar to how exiftool determines the image type as ARW, we are implementing a check here
+  # https://github.com/exiftool/exiftool/blob/e969456372fbaf4b980fea8bb094d71033ac8bf7/lib/Image/ExifTool/Exif.pm#L929
+  def arw?(exif_data)
+    exif_data.compression == 6 && exif_data.new_subfile_type == 1 && exif_data.make == 'SONY'
+  end
+
   FormatParser.register_parser new, natures: :image, formats: :tif
 end

data/spec/parsers/moov_parser_spec.rb

@@ -108,4 +108,24 @@ describe FormatParser::MOOVParser do
   it 'provides filename hints' do
     expect(subject).to be_likely_match('file.m4v')
   end
+
+  it 'reads correctly the video dimensions' do
+    mov_path = fixtures_dir + '/MOOV/MOV/Test_Dimensions.mov'
+
+    result = subject.call(File.open(mov_path, 'rb'))
+
+    expect(result).not_to be_nil
+    expect(result.nature).to eq(:video)
+    expect(result.format).to eq(:mov)
+    expect(result.width_px).to eq(640)
+    expect(result.height_px).to eq(360)
+  end
+
+  it 'does not raise error when a meta atom has size 0' do
+    mov_path = fixtures_dir + '/MOOV/MOV/Test_Meta_Atom_With_Size_Zero.mov'
+
+    result = subject.call(File.open(mov_path, 'rb'))
+    expect(result).not_to be_nil
+    expect(result.format).to eq(:mov)
+  end
 end

data/spec/parsers/mp3_parser_spec.rb

@@ -15,6 +15,26 @@ describe FormatParser::MP3Parser do
     expect(parsed.media_duration_seconds).to be_within(0.1).of(0.836)
   end
 
+  it 'reads the Xing header without raising errors' do
+    fpath = fixtures_dir + '/MP3/test_xing_header.mp3'
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed).not_to be_nil
+
+    expect(parsed.nature).to eq(:audio)
+    expect(parsed.format).to eq(:mp3)
+    expect(parsed.num_audio_channels).to eq(2)
+    expect(parsed.audio_sample_rate_hz).to eq(48000)
+    expect(parsed.intrinsics).not_to be_nil
+    expect(parsed.media_duration_seconds).to be_within(0.1).of(0.0342)
+  end
+
+  it 'does not misdetect a PNG' do
+    fpath = fixtures_dir + '/PNG/anim.png'
+    parsed = subject.call(File.open(fpath, 'rb'))
+    expect(parsed).to be_nil
+  end
+
   describe 'title/artist/album attributes' do
     let(:parsed) { subject.call(File.open(fpath, 'rb')) }
 
@@ -67,7 +87,7 @@ describe FormatParser::MP3Parser do
 
     large_syncsfe_size = [ID3Tag::SynchsafeInteger.encode(more_bytes_than_permitted)].pack('N')
     prepped = StringIO.new(
-      'ID3' + "\x43\x00".b + "\x00".b + large_syncsfe_size + gunk
+      'ID3' + "\x03\x00".b + "\x00".b + large_syncsfe_size + gunk
     )
 
     expect(ID3Tag).not_to receive(:read)
@@ -90,6 +110,14 @@ describe FormatParser::MP3Parser do
     expect(parsed.title).to eq('test')
   end
 
+  it 'reads the mpeg frames correctly' do
+    fpath = fixtures_dir + '/MP3/test_read_frames.mp3'
+
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed.audio_sample_rate_hz). to eq(48000)
+  end
+
   it 'parses the Cassy MP3' do
     fpath = fixtures_dir + '/MP3/Cassy.mp3'
     parsed = subject.call(File.open(fpath, 'rb'))
@@ -130,6 +158,14 @@ describe FormatParser::MP3Parser do
     }.to raise_error(FormatParser::IOUtils::InvalidRead)
   end
 
+  it 'supports id3 v2.4.x' do
+    fpath = fixtures_dir + '/MP3/id3v24.mp3'
+
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed.artist). to eq('wetransfer')
+  end
+
   describe '#as_json' do
     it 'converts all hash keys to string when stringify_keys: true' do
       fpath = fixtures_dir + '/MP3/Cassy.mp3'

data/spec/parsers/tiff_parser_spec.rb

@@ -47,12 +47,15 @@ describe FormatParser::TIFFParser do
     expect(parsed.intrinsics[:exif]).not_to be_nil
   end
 
-  it 'correctly extracts dimensions for a Sony ARW fixture' do
+  it 'parses Sony ARW fixture as arw format file' do
     arw_path = fixtures_dir + '/ARW/RAW_SONY_ILCE-7RM2.ARW'
 
     parsed = subject.call(File.open(arw_path, 'rb'))
 
     expect(parsed).not_to be_nil
+    expect(parsed.nature).to eq(:image)
+    expect(parsed.format).to eq(:arw)
+
     expect(parsed.width_px).to eq(7952)
     expect(parsed.height_px).to eq(5304)
     expect(parsed.intrinsics[:exif]).not_to be_nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.23.1
+  version: 0.25.1
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-14 00:00:00.000000000 Z
+date: 2020-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -34,7 +34,7 @@ dependencies:
         version: '1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.7
+        version: 1.3.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -44,21 +44,21 @@ dependencies:
         version: '1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.7
+        version: 1.3.8
 - !ruby/object:Gem::Dependency
   name: id3tag
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '0.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '0.14'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -292,7 +292,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A library for efficient parsing of file metadata