format_parser 0.24.0 → 0.25.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cef6e8b3f4c3914ef022e7ff0e92d38357670315c0804efe8a2967f7f4228f53
-  data.tar.gz: '09a817b4fc688a3695800f4602a5901613b591d7a30ba26b86988720da7e13a9'
+  metadata.gz: 290b2369d2fe089202a76ab9d7b659f8de58fa44b9a40718130105ae7026036a
+  data.tar.gz: 53e983a8639cc42377ab50d7364b5099cf1f3308f8108f92b6194040546ea2e8
 SHA512:
-  metadata.gz: 979804c2a7381d09e1f4952396321e185a917587d44409b1f7d45be0e05b466601eb7281e3d463785c6c18d79ba48ff3858bcd60c0f6791e2d394c4fa40ed684
-  data.tar.gz: a58cd8cab20fa8b8b7886a36479bd03fb6f5a4afbf9435d09318da502de50f6f257e45b012a56c79b4309dbaaf96feedb6b2b12016979bffa1336c528a591e3b
+  metadata.gz: 5f58620ab165b1c47a8a18fe82081d48cb0b285b4f9638146c31d1bb2f839247b36750a7f85be5a9ea14b8db3fc2ca175b86ab4f8b29f87bd1ea10caed57746c
+  data.tar.gz: b9e87723a7cc1d5ecf04a23c28cbfc433c120337275aa51d76396a9a1371bb1b683b22c18ab4e84d8870a76048e26f7c5d7f90b4b5f9b868bf5a0dcb6771640c

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 0.25.1
+* MOV: Fix error "negative length"
+* MOV: Fix reading dimensions in multi-track files
+* MP3: Fix parse of the Xing header to not raise errors
+
+## 0.25.0
+* MP3: add suport to id3 v2.4.x
+* JPEG: Update gem exifr to 1.3.8 to fix a bug
+
+## 0.24.2
+* Update gem id3tag to 0.14.0 to fix MP3 issues
+
+## 0.24.1
+* Fix MP3 frames reading to jump correctly to the next bytes
+
 ## 0.24.0
 * The TIFF parser will now return :arw as format for Sony ARW files insted of :tif so that the caller can decide whether it
   wants to deal with RAW processing or not

data/format_parser.gemspec

@@ -31,8 +31,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'ks', '~> 0.0'
-  spec.add_dependency 'exifr', '~> 1', '>= 1.3.7'
-  spec.add_dependency 'id3tag', '~> 0.13'
+  spec.add_dependency 'exifr', '~> 1', '>= 1.3.8'
+  spec.add_dependency 'id3tag', '~> 0.14'
   spec.add_dependency 'faraday', '~> 0.13'
   spec.add_dependency 'measurometer', '~> 1'
 

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.24.0'
+  VERSION = '0.25.2'
 end

data/lib/parsers/moov_parser.rb

@@ -38,14 +38,8 @@ class FormatParser::MOOVParser
     ftyp_atom = decoder.find_first_atom_by_path(atom_tree, 'ftyp')
     file_type = ftyp_atom.field_value(:major_brand)
 
-    width = nil
-    height = nil
-
     # Try to find the width and height in the tkhd
-    if tkhd = decoder.find_first_atom_by_path(atom_tree, 'moov', 'trak', 'tkhd')
-      width = tkhd.field_value(:track_width).first
-      height = tkhd.field_value(:track_height).first
-    end
+    width, height = parse_dimensions(decoder, atom_tree)
 
     # Try to find the "topmost" duration (respecting edits)
     if mdhd = decoder.find_first_atom_by_path(atom_tree, 'moov', 'mvhd')
@@ -78,6 +72,31 @@ class FormatParser::MOOVParser
     FTYP_MAP.fetch(file_type.downcase, :mov)
   end
 
+  # The dimensions are located in tkhd atom, but in some files it is necessary
+  # to get it below the video track, because it can have other tracks such as
+  # audio which does not have the dimensions.
+  # More details in https://developer.apple.com/library/archive/documentation/QuickTime/QTFF/QTFFChap2/qtff2.html#//apple_ref/doc/uid/TP40000939-CH204-DontLinkElementID_147
+  #
+  # Returns [width, height] if the dimension is found
+  # Returns [nil, nil] if the dimension is not found
+  def parse_dimensions(decoder, atom_tree)
+    video_trak_atom = decoder.find_video_trak_atom(atom_tree)
+
+    tkhd = begin
+      if video_trak_atom
+        decoder.find_first_atom_by_path([video_trak_atom], 'trak', 'tkhd')
+      else
+        decoder.find_first_atom_by_path(atom_tree, 'moov', 'trak', 'tkhd')
+      end
+    end
+
+    if tkhd
+      [tkhd.field_value(:track_width).first, tkhd.field_value(:track_height).first]
+    else
+      [nil, nil]
+    end
+  end
+
   # An MPEG4/MOV/M4A will start with the "ftyp" atom. The atom must have a length
   # of at least 8 (to accomodate the atom size and the atom type itself) plus the major
   # and minor version fields. If we cannot find it we can be certain this is not our file.

data/lib/parsers/moov_parser/decoder.rb

@@ -1,6 +1,7 @@
 # Handles decoding of MOV/MPEG4 atoms/boxes in a stream. Will recursively
 # read atoms and parse their data fields if applicable. Also contains
 # a few utility functions for finding atoms in a list etc.
+# To know more about Atoms: https://developer.apple.com/library/archive/documentation/QuickTime/QTFF/QTFFChap2/qtff2.html
 class FormatParser::MOOVParser::Decoder
   include FormatParser::IOUtils
 
@@ -47,6 +48,34 @@ class FormatParser::MOOVParser::Decoder
     find_first_atom_by_path(requisite.children || [], *atom_types)
   end
 
+  def find_atoms_by_path(atoms, atom_types)
+    type_to_find = atom_types.shift
+    requisites = atoms.select { |e| e.atom_type == type_to_find }
+
+    # Return if we found our match
+    return requisites if atom_types.empty?
+
+    # Return nil if we didn't find the match at this nesting level
+    return unless requisites
+
+    # ...otherwise drill further down
+    find_atoms_by_path(requisites.flat_map(&:children).compact || [], atom_types)
+  end
+
+  # A file can have multiple tracks. To identify the type it is necessary to check
+  # the fields `omponent_subtype` in hdlr atom under the trak atom
+  # More details in https://developer.apple.com/library/archive/documentation/QuickTime/QTFF/QTFFChap2/qtff2.html#//apple_ref/doc/uid/TP40000939-CH204-DontLinkElementID_147
+  def find_video_trak_atom(atoms)
+    trak_atoms = find_atoms_by_path(atoms, ['moov', 'trak'])
+
+    return if trak_atoms.empty?
+
+    trak_atoms.find do |trak_atom|
+      hdlr_atom = find_first_atom_by_path([trak_atom], 'trak', 'mdia', 'hdlr')
+      hdlr_atom.atom_fields[:component_type] == 'mhlr' && hdlr_atom.atom_fields[:component_subtype] == 'vide'
+    end
+  end
+
   def parse_ftyp_atom(io, atom_size)
     # Subtract 8 for the atom_size+atom_type,
     # and 8 once more for the major_brand and minor_version. The remaining
@@ -194,6 +223,8 @@ class FormatParser::MOOVParser::Decoder
   end
 
   def parse_meta_atom(io, atom_size)
+    return if atom_size == 0 # this atom can be empty
+
     parse_hdlr_atom(io, atom_size)
   end
 

data/lib/parsers/mp3_parser.rb

@@ -20,13 +20,14 @@ class FormatParser::MP3Parser
 
   # We limit the number of MPEG frames we scan
   # to obtain our duration estimation
-  MAX_FRAMES_TO_SCAN = 128
+  MAX_FRAMES_TO_SCAN = 500
 
   # Default frame size for mp3
   SAMPLES_PER_FRAME = 1152
 
   # For some edge cases
   ZIP_LOCAL_ENTRY_SIGNATURE = "PK\x03\x04\x14\x00".b
+  PNG_HEADER_BYTES = [137, 80, 78, 71, 13, 10, 26, 10].pack('C*')
 
   # Wraps the Tag object returned by ID3Tag in such
   # a way that a usable JSON representation gets
@@ -60,8 +61,12 @@ class FormatParser::MP3Parser
     # To avoid having that happen, we check for the PKZIP signature -
     # local entry header signature - at the very start of the file.
     # If the file is too small safe_read will fail too and the parser
-    # will terminate here.
-    return if safe_read(io, 6) == ZIP_LOCAL_ENTRY_SIGNATURE
+    # will terminate here. Same with PNGs. In the future
+    # we should implement "confidence" for MP3 as of all our formats
+    # it is by far the most lax.
+    header = safe_read(io, 8)
+    return if header.start_with?(ZIP_LOCAL_ENTRY_SIGNATURE)
+    return if header.start_with?(PNG_HEADER_BYTES)
 
     # Read all the ID3 tags (or at least attempt to)
     io.seek(0)
@@ -131,27 +136,28 @@ class FormatParser::MP3Parser
   # if you have a minute. https://pypi.python.org/pypi/tinytag
   def parse_mpeg_frames(io)
     mpeg_frames = []
+    bytes_to_read = 4
 
     MAX_FRAMES_TO_SCAN.times do |frame_i|
       # Read through until we can latch onto the 11 sync bits. Read in 4-byte
       # increments to save on read() calls
-      data = io.read(4)
+      data = io.read(bytes_to_read)
 
       # If we are at EOF - stop iterating
-      break unless data && data.bytesize == 4
+      break unless data && data.bytesize == bytes_to_read
 
       # Look for the sync pattern. It can be either the last byte being 0xFF,
       # or any of the 2 bytes in sequence being 0xFF and > 0xF0.
       four_bytes = data.unpack('C4')
       seek_jmp = sync_bytes_offset_in_4_byte_seq(four_bytes)
       if seek_jmp > 0
-        io.seek(io.pos + seek_jmp)
+        io.seek(io.pos - bytes_to_read + seek_jmp)
         next
       end
 
       # Once we are past that stage we have latched onto a sync frame header
       sync, conf, bitrate_freq, rest = four_bytes
-      frame_detail = parse_mpeg_frame_header(io.pos - 4, sync, conf, bitrate_freq, rest)
+      frame_detail = parse_mpeg_frame_header(io.pos - bytes_to_read, sync, conf, bitrate_freq, rest)
       mpeg_frames << frame_detail
 
       # There might be a xing header in the first frame that contains
@@ -166,7 +172,7 @@ class FormatParser::MP3Parser
         end
       end
       if frame_detail.frame_length > 1 # jump over current frame body
-        io.seek(io.pos + frame_detail.frame_length - 4)
+        io.seek(io.pos + frame_detail.frame_length - bytes_to_read)
       end
     end
     [nil, mpeg_frames]
@@ -243,16 +249,16 @@ class FormatParser::MP3Parser
     io.seek(xing_offset + 4) # Include the length of "Xing" itself
 
     # https://www.codeproject.com/Articles/8295/MPEG-Audio-Frame-Header#XINGHeader
-    header_flags, _ = io.read(4).unpack('s>s>')
+    header_flags, _ = io.read(4).unpack('i>')
     frames = byte_count = toc = vbr_scale = nil
 
-    frames = io.read(4).unpack('N1').first if header_flags & 1 # FRAMES FLAG
+    frames = io.read(4).unpack('N1').first if header_flags & 1 != 0   # FRAMES FLAG
 
-    byte_count = io.read(4).unpack('N1').first if header_flags & 2 # BYTES FLAG
+    byte_count = io.read(4).unpack('N1').first if header_flags & 2 != 0   # BYTES FLAG
 
-    toc = io.read(100).unpack('C100') if header_flags & 4 # TOC FLAG
+    toc = io.read(100).unpack('C100') if header_flags & 4 != 0   # TOC FLAG
 
-    vbr_scale = io.read(4).unpack('N1').first if header_flags & 8 # VBR SCALE FLAG
+    vbr_scale = io.read(4).unpack('N1').first if header_flags & 8 != 0   # VBR SCALE FLAG
 
     VBRHeader.new(frames: frames, byte_count: byte_count, toc_entries: toc, vbr_scale: vbr_scale)
   end

data/lib/parsers/mp3_parser/id3_extraction.rb

@@ -1,6 +1,8 @@
 module FormatParser::MP3Parser::ID3Extraction
   ID3V1_TAG_SIZE_BYTES = 128
-  ID3V2_TAG_VERSIONS = ["\x43\x00".b, "\x03\x00".b, "\x02\x00".b]
+  # it supports 2.4.x, 2.3.x, 2.2.x which are supported by the gem id3tag
+  # see https://id3.org/Developer%20Information for more details of each version
+  ID3V2_MINOR_TAG_VERSIONS = [2, 3, 4]
   MAX_SIZE_FOR_ID3V2 = 1 * 1024 * 1024
 
   extend FormatParser::IOUtils
@@ -22,7 +24,7 @@ module FormatParser::MP3Parser::ID3Extraction
     io.seek(0) # Only support header ID3v2
     header = parse_id3_v2_header(io)
     return unless header[:tag] == 'ID3' && header[:size] > 0
-    return unless ID3V2_TAG_VERSIONS.include?(header[:version])
+    return unless ID3V2_MINOR_TAG_VERSIONS.include?(header[:version].unpack('C').first)
 
     id3_tag_size = io.pos + header[:size]
 

data/spec/parsers/moov_parser_spec.rb

@@ -108,4 +108,24 @@ describe FormatParser::MOOVParser do
   it 'provides filename hints' do
     expect(subject).to be_likely_match('file.m4v')
   end
+
+  it 'reads correctly the video dimensions' do
+    mov_path = fixtures_dir + '/MOOV/MOV/Test_Dimensions.mov'
+
+    result = subject.call(File.open(mov_path, 'rb'))
+
+    expect(result).not_to be_nil
+    expect(result.nature).to eq(:video)
+    expect(result.format).to eq(:mov)
+    expect(result.width_px).to eq(640)
+    expect(result.height_px).to eq(360)
+  end
+
+  it 'does not raise error when a meta atom has size 0' do
+    mov_path = fixtures_dir + '/MOOV/MOV/Test_Meta_Atom_With_Size_Zero.mov'
+
+    result = subject.call(File.open(mov_path, 'rb'))
+    expect(result).not_to be_nil
+    expect(result.format).to eq(:mov)
+  end
 end

data/spec/parsers/mp3_parser_spec.rb

@@ -15,6 +15,26 @@ describe FormatParser::MP3Parser do
     expect(parsed.media_duration_seconds).to be_within(0.1).of(0.836)
   end
 
+  it 'reads the Xing header without raising errors' do
+    fpath = fixtures_dir + '/MP3/test_xing_header.mp3'
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed).not_to be_nil
+
+    expect(parsed.nature).to eq(:audio)
+    expect(parsed.format).to eq(:mp3)
+    expect(parsed.num_audio_channels).to eq(2)
+    expect(parsed.audio_sample_rate_hz).to eq(48000)
+    expect(parsed.intrinsics).not_to be_nil
+    expect(parsed.media_duration_seconds).to be_within(0.1).of(0.0342)
+  end
+
+  it 'does not misdetect a PNG' do
+    fpath = fixtures_dir + '/PNG/anim.png'
+    parsed = subject.call(File.open(fpath, 'rb'))
+    expect(parsed).to be_nil
+  end
+
   describe 'title/artist/album attributes' do
     let(:parsed) { subject.call(File.open(fpath, 'rb')) }
 
@@ -67,7 +87,7 @@ describe FormatParser::MP3Parser do
 
     large_syncsfe_size = [ID3Tag::SynchsafeInteger.encode(more_bytes_than_permitted)].pack('N')
     prepped = StringIO.new(
-      'ID3' + "\x43\x00".b + "\x00".b + large_syncsfe_size + gunk
+      'ID3' + "\x03\x00".b + "\x00".b + large_syncsfe_size + gunk
     )
 
     expect(ID3Tag).not_to receive(:read)
@@ -90,6 +110,14 @@ describe FormatParser::MP3Parser do
     expect(parsed.title).to eq('test')
   end
 
+  it 'reads the mpeg frames correctly' do
+    fpath = fixtures_dir + '/MP3/test_read_frames.mp3'
+
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed.audio_sample_rate_hz). to eq(48000)
+  end
+
   it 'parses the Cassy MP3' do
     fpath = fixtures_dir + '/MP3/Cassy.mp3'
     parsed = subject.call(File.open(fpath, 'rb'))
@@ -130,6 +158,14 @@ describe FormatParser::MP3Parser do
     }.to raise_error(FormatParser::IOUtils::InvalidRead)
   end
 
+  it 'supports id3 v2.4.x' do
+    fpath = fixtures_dir + '/MP3/id3v24.mp3'
+
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed.artist). to eq('wetransfer')
+  end
+
   describe '#as_json' do
     it 'converts all hash keys to string when stringify_keys: true' do
       fpath = fixtures_dir + '/MP3/Cassy.mp3'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.24.0
+  version: 0.25.2
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-15 00:00:00.000000000 Z
+date: 2020-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -34,7 +34,7 @@ dependencies:
         version: '1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.7
+        version: 1.3.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -44,21 +44,21 @@ dependencies:
         version: '1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.7
+        version: 1.3.8
 - !ruby/object:Gem::Dependency
   name: id3tag
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '0.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '0.14'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -292,7 +292,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A library for efficient parsing of file metadata