format_parser 2.0.0 → 2.2.0

data/lib/parsers/jpeg_parser.rb

@@ -178,7 +178,7 @@ class FormatParser::JPEGParser
 
   def skip_frame
     length = read_short - 2
-    safe_skip(@buf, length)
+    skip_bytes(length)
   end
 
   FormatParser.register_parser self, natures: :image, formats: :jpg, priority: 0

data/lib/parsers/moov_parser/decoder.rb

@@ -63,7 +63,7 @@ class FormatParser::MOOVParser::Decoder
   end
 
   # A file can have multiple tracks. To identify the type it is necessary to check
-  # the fields `omponent_subtype` in hdlr atom under the trak atom
+  # the fields `component_subtype` in hdlr atom under the trak atom
   # More details in https://developer.apple.com/library/archive/documentation/QuickTime/QTFF/QTFFChap2/qtff2.html#//apple_ref/doc/uid/TP40000939-CH204-DontLinkElementID_147
   def find_video_trak_atom(atoms)
     trak_atoms = find_atoms_by_path(atoms, ['moov', 'trak'])

data/lib/parsers/moov_parser.rb

@@ -6,9 +6,9 @@ class FormatParser::MOOVParser
   # we can reasonably call "file type" (something
   # usable as a filename extension)
   FTYP_MAP = {
-    'qt  ' => :mov,
-    'mp4 ' => :mp4,
     'm4a ' => :m4a,
+    'mp4 ' => :mp4,
+    'qt  ' => :mov,
   }
 
   # https://tools.ietf.org/html/rfc4337#section-2
@@ -18,7 +18,7 @@ class FormatParser::MOOVParser
   MP4_MIXED_MIME_TYPE = 'video/mp4'
 
   def likely_match?(filename)
-    filename =~ /\.(mov|m4a|ma4|mp4|aac|m4v)$/i
+    filename =~ /\.(m4a|m4v|ma4|mov|mp4)$/i
   end
 
   def call(io)
@@ -42,7 +42,8 @@ class FormatParser::MOOVParser
     end
 
     ftyp_atom = decoder.find_first_atom_by_path(atom_tree, 'ftyp')
-    file_type = ftyp_atom.field_value(:major_brand)
+    file_type = ftyp_atom&.field_value(:major_brand)
+    format = format_from_moov_type(file_type)
 
     # Try to find the width and height in the tkhd
     width, height = parse_dimensions(decoder, atom_tree)
@@ -55,17 +56,16 @@ class FormatParser::MOOVParser
     end
 
     # M4A only contains audio, while MP4 and friends can contain video.
-    fmt = format_from_moov_type(file_type)
-    if fmt == :m4a
+    if format == :m4a
       FormatParser::Audio.new(
-        format: format_from_moov_type(file_type),
+        format: format,
         media_duration_seconds: media_duration_s,
         content_type: MP4_AU_MIME_TYPE,
         intrinsics: atom_tree,
       )
     else
       FormatParser::Video.new(
-        format: format_from_moov_type(file_type),
+        format: format,
         width_px: width,
         height_px: height,
         frame_rate: parse_time_to_sample_atom(decoder, atom_tree)&.truncate(2),
@@ -161,5 +161,5 @@ class FormatParser::MOOVParser
     end
   end
 
-  FormatParser.register_parser new, natures: :video, formats: FTYP_MAP.values, priority: 3
+  FormatParser.register_parser new, natures: [:audio, :video], formats: FTYP_MAP.values, priority: 3
 end

data/lib/parsers/mp3_parser.rb

@@ -1,4 +1,3 @@
-require 'ks'
 require 'id3tag'
 
 class FormatParser::MP3Parser
@@ -6,13 +5,13 @@ class FormatParser::MP3Parser
 
   require_relative 'mp3_parser/id3_extraction'
 
-  class MPEGFrame < Ks.strict(:offset_in_file, :mpeg_id, :channels, :sample_rate, :frame_length, :frame_bitrate)
+  class MPEGFrame < Struct.new(:offset_in_file, :mpeg_id, :channels, :sample_rate, :frame_length, :frame_bitrate, keyword_init: true)
   end
 
-  class VBRHeader < Ks.strict(:frames, :byte_count, :toc_entries, :vbr_scale)
+  class VBRHeader < Struct.new(:frames, :byte_count, :toc_entries, :vbr_scale, keyword_init: true)
   end
 
-  class MP3Info < Ks.strict(:duration_seconds, :num_channels, :sampling_rate)
+  class MP3Info < Struct.new(:duration_seconds, :num_channels, :sampling_rate, keyword_init: true)
   end
 
   class InvalidDeepFetch < KeyError

data/lib/parsers/webp_parser.rb

@@ -54,7 +54,7 @@ class FormatParser::WebpParser
     # Encoded as a single VP8 key frame - a 10-byte uncompressed chunk followed by 2+ partitions of compressed data.
     # The first 6 bytes of this chunk contains information that is mostly relevant when using VP8 as a video
     # compression format, and can be ignored.
-    safe_skip(@buf, 6)
+    skip_bytes(6)
 
     # The subsequent 4 bytes contain the image width and height, respectively, as 16-bit unsigned little endian
     # integers.
@@ -64,7 +64,7 @@ class FormatParser::WebpParser
 
   def read_lossless_data
     # There is a single byte signature, 0x2F, that we can disregard.
-    safe_skip(@buf, 1)
+    skip_bytes(1)
 
     # The subsequent 4 bytes contain the image width and height, respectively, as 14-bit unsigned little endian
     # integers (minus one). The 4 remaining bits consist of a 1-bit flag indicating whether alpha is used, and a 3-bit
@@ -100,7 +100,7 @@ class FormatParser::WebpParser
 
     # The flags are followed by three reserved bytes of zeros, and then by the width and height, respectively - each
     # occupying three bytes and each one less than the actual canvas measurements.
-    safe_skip(@buf, 3)
+    skip_bytes(3)
     dimensions = safe_read(@buf, 6).unpack('VS')
     width = (dimensions[0] & 0xffffff) + 1
     height = (dimensions[0] >> 24 | dimensions[1] << 8 & 0xffffff) + 1
@@ -134,7 +134,7 @@ class FormatParser::WebpParser
       end
 
       # Padding byte of 0 added if chunk size is odd.
-      safe_skip(@buf, 1) if chunk_size.odd?
+      skip_bytes(1) if chunk_size.odd?
 
       case fourcc
       when 'EXIF'
@@ -155,9 +155,9 @@ class FormatParser::WebpParser
         intrinsics[:xmp] ||= safe_read(@buf, chunk_size)
       when 'ANMF'
         num_frames += 1 if image.has_multiple_frames
-        safe_skip(@buf, chunk_size)
+        skip_bytes(chunk_size)
       else
-        safe_skip(@buf, chunk_size)
+        skip_bytes(chunk_size)
       end
     end
 

data/spec/format_parser_spec.rb

@@ -177,14 +177,16 @@ describe FormatParser do
     it 'sorts the parsers by priority and name' do
       parsers = FormatParser.parsers_for(
         [:audio, :image],
-        [:cr2, :dpx, :fdx, :flac, :gif, :jpg, :mov, :mp4, :m4a, :mp3, :mpg, :mpeg, :ogg, :png, :tif, :wav]
+        [:cr2, :cr3, :dpx, :fdx, :flac, :gif, :jpg, :mov, :mp4, :m4a, :mp3, :mpg, :mpeg, :ogg, :png, :tif, :wav]
       )
 
       expect(parsers.map { |parser| parser.class.name }).to eq([
         'FormatParser::GIFParser',
         'Class',
         'FormatParser::PNGParser',
+        'FormatParser::MOOVParser',
         'FormatParser::CR2Parser',
+        'FormatParser::CR3Parser',
         'FormatParser::DPXParser',
         'FormatParser::FLACParser',
         'FormatParser::MP3Parser',

data/spec/parsers/cr3_parser_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+
+describe FormatParser::CR3Parser do
+  it 'should match valid CR3 file extensions' do
+    valid_extensions = %w[cr3 cR3 Cr3 CR3]
+    valid_extensions.each { |extension| expect(subject.likely_match?("foo.#{extension}")).to be_truthy }
+  end
+
+  it 'should not match invalid CR3 file extensions' do
+    invalid_filenames = ['', 'foo', 'cr3', 'foocr3', 'foo.cr2', 'foo.cr3.bar']
+    invalid_filenames.each { |filename| expect(subject.likely_match?(filename)).to be_falsey }
+  end
+
+  it 'should not parse a file that does not match the CR3 definition' do
+    # MOV files are closely related to CR3 files (both extend the ISO Base File Format), so this is a decent edge case
+    # to ensure only true CR3 files are being parsed.
+    result = subject.call(File.open(fixtures_dir + '/MOOV/MOV/Test_Dimensions.mov'))
+    expect(result).to be_nil
+  end
+
+  it 'should return nil if no CMT1 atom is present' do
+    # This is a MOV file with the ftyp header modified to masquerade as a CR3 file. It is therefore missing the
+    # CR3-specific CMT1 atom containing the image metadata.
+    result = subject.call(File.open(fixtures_dir + '/CR3/invalid'))
+    expect(result).to be_nil
+  end
+
+  Dir.glob(fixtures_dir + '/CR3/*.cr3').sort.each do |file_path|
+    it "successfully parses #{File.basename(file_path)}" do
+      result = subject.call(File.open(file_path, 'rb'))
+
+      expect(result).not_to be_nil
+      expect(result.nature).to eq(:image)
+      expect(result.width_px).to be > 0
+      expect(result.height_px).to be > 0
+      expect(result.content_type).to eq('image/x-canon-cr3')
+      expect(result.intrinsics).not_to be_nil
+    end
+  end
+
+  it 'parses the necessary metadata from a CR3 file' do
+    file_path = fixtures_dir + '/CR3/Canon EOS R10 (RAW).CR3'
+
+    result = subject.call(File.open(file_path, 'rb'))
+    expect(result.nature).to eq(:image)
+    expect(result.width_px).to eq(6000)
+    expect(result.height_px).to eq(4000)
+    expect(result.orientation).to eq(:top_left)
+    expect(result.display_width_px).to eq(6000)
+    expect(result.display_height_px).to eq(4000)
+    expect(result.content_type).to eq('image/x-canon-cr3')
+    expect(result.intrinsics).not_to be_nil
+    expect(result.intrinsics[:atom_tree]).not_to be_nil
+    expect(result.intrinsics[:exif]).not_to be_nil
+    expect(result.intrinsics[:exif][:image_length]).to eq(result.height_px)
+    expect(result.intrinsics[:exif][:image_width]).to eq(result.width_px)
+  end
+end

data/spec/parsers/iso_base_media_file_format/decoder_spec.rb

@@ -0,0 +1,242 @@
+require 'spec_helper'
+
+describe FormatParser::ISOBaseMediaFileFormat::Decoder do
+  context 'when build_atom_tree is called' do
+    context 'with no io' do
+      it 'raises an error' do
+        expect { subject.build_atom_tree(0x0) }.to raise_error(/IO missing - supply a valid IO object/)
+      end
+    end
+
+    context('with a max_read smaller than the length of the IO') do
+      let(:io) do
+        # moov
+        # moov
+        input = [0x8].pack('N') + 'moov' + [0x8].pack('N') + 'moov'
+        StringIO.new(input)
+      end
+
+      it 'stops building the tree' do
+        expect(subject.build_atom_tree(0x0, io).length).to eq(0)
+        expect(io.pos).to eq(0)
+
+        expect(subject.build_atom_tree(0x8, io).length).to eq(1)
+        expect(io.pos).to eq(0x8)
+        io.seek(0)
+
+        expect(subject.build_atom_tree(0x10, io).length).to eq(2)
+        expect(io.pos).to eq(0x10)
+      end
+    end
+
+    context 'when parsing an unknown atom' do
+      let(:io) do
+        # foo
+        # |-> moov
+        input = [0x14].pack('N') + 'foo ' + [0x1].pack('N') + [0x8].pack('N') + 'moov'
+        StringIO.new(input)
+      end
+
+      it 'parses only the type, position and size, and skips any fields and children' do
+        result = subject.build_atom_tree(0xFF, io)
+        expect(result.length).to eq(1)
+        expect(io.pos).to eq(0x14)
+
+        foo_atom = result[0]
+        expect(foo_atom.type).to eq('foo ')
+        expect(foo_atom.position).to eq(0)
+        expect(foo_atom.size).to eq(0x14)
+        expect(foo_atom.fields).to eq({})
+        expect(foo_atom.children).to eq([])
+      end
+    end
+
+    context 'when parsing a container atom' do
+      let(:io) do
+        # moov
+        # |-> foo
+        # |-> bar
+        input = [0x18].pack('N') + 'moov' + [0x8].pack('N') + 'foo ' + [0x8].pack('N') + 'bar '
+        StringIO.new(input)
+      end
+
+      it 'parses type, position, size and children' do
+        result = subject.build_atom_tree(0xFF, io)
+        expect(result.length).to eq(1)
+        expect(io.pos).to eq(0x18)
+
+        moov_atom = result[0]
+        expect(moov_atom.type).to eq('moov')
+        expect(moov_atom.position).to eq(0)
+        expect(moov_atom.size).to eq(0x18)
+        expect(moov_atom.fields).to eq({})
+        expect(moov_atom.children.length).to eq(2)
+      end
+    end
+
+    context 'when parsing an empty atom' do
+      let(:io) do
+        # nmhd
+        # |-> foo
+        input = [0x18].pack('N') + 'nmhd' + [0x1].pack('c') + 'fla' + [0x2].pack('N') + [0x8].pack('N') + 'foo '
+        StringIO.new(input)
+      end
+
+      it 'parses type, position, size, version and flags, and skips any other fields or children' do
+        result = subject.build_atom_tree(0xFF, io)
+        expect(result.length).to eq(1)
+        expect(io.pos).to eq(0x18)
+
+        nmhd_atom = result[0]
+        expect(nmhd_atom.type).to eq('nmhd')
+        expect(nmhd_atom.position).to eq(0)
+        expect(nmhd_atom.size).to eq(0x18)
+        expect(nmhd_atom.fields).to include({
+          version: 1,
+          flags: 'fla'
+        })
+        expect(nmhd_atom.children).to eq([])
+      end
+    end
+
+    context 'when parsing a uuid atom' do
+      let(:usertype) { '90f7c66ec2db476b977461e796f0dd4b' }
+      let(:io) do
+        input = [0x20].pack('N') + 'uuid' + [usertype].pack('H*') + [0x8].pack('N') + 'foo '
+        StringIO.new(input)
+      end
+
+      it 'parses type, position, size and usertype, and skips any other fields or children' do
+        # uuid
+        # |-> foo
+        result = subject.build_atom_tree(0xFF, io)
+        expect(result.length).to eq(1)
+        expect(io.pos).to eq(0x20)
+
+        nmhd_atom = result[0]
+        expect(nmhd_atom.type).to eq('uuid')
+        expect(nmhd_atom.position).to eq(0)
+        expect(nmhd_atom.size).to eq(0x20)
+        expect(nmhd_atom.fields).to include({
+          usertype: usertype,
+        })
+        expect(nmhd_atom.children).to eq([])
+      end
+    end
+  end
+end
+
+describe FormatParser::ISOBaseMediaFileFormat::Decoder::Atom do
+  context 'when initialized' do
+    context 'without fields and/or children' do
+      subject { described_class.new('foo', 0, 0) }
+
+      it 'sets them as an empty array/hash' do
+        expect(subject.type).to eq('foo')
+        expect(subject.position).to eq(0)
+        expect(subject.size).to eq(0)
+        expect(subject.fields).to eq({})
+        expect(subject.children).to eq([])
+      end
+    end
+
+    context 'with fields and/or children' do
+      let(:fields) { { foo: 1, bar: 'bar' } }
+      let(:children) { [described_class.new('bar', 0, 0)] }
+
+      subject { described_class.new('foo', 0, 0, fields, children) }
+
+      it 'sets them correctly' do
+        expect(subject.type).to eq('foo')
+        expect(subject.position).to eq(0)
+        expect(subject.size).to eq(0)
+        expect(subject.fields).to eq(fields)
+        expect(subject.children).to eq(children)
+      end
+    end
+  end
+
+  context 'when find_first_descendent is called' do
+    context 'with no children' do
+      subject { described_class.new('root', 0, 0) }
+      it 'returns nil' do
+        expect(subject.find_first_descendent(%w[root foo])).to be_nil
+      end
+    end
+
+    context 'with no descendents of the given type(s)' do
+      subject do
+        described_class.new('root', 0, 0, nil, [
+          described_class.new('foo', 0, 0),
+          described_class.new('bar', 0, 0, nil, [
+            described_class.new('baz', 0, 0)
+          ])
+        ])
+      end
+
+      it 'returns nil' do
+        expect(subject.find_first_descendent(%w[root qux])).to be_nil
+      end
+    end
+
+    context 'with multiple descendents of the given type(s)' do
+      let(:descendent) { described_class.new('bar', 0, 0) }
+
+      subject do
+        described_class.new('root', 0, 0, nil, [
+          described_class.new('foo', 0, 0, nil, [
+            descendent
+          ]),
+          described_class.new('bar', 0, 0),
+        ])
+      end
+
+      it 'returns the first relevant descendent in order of appearance' do
+        expect(subject.find_first_descendent(%w[bar])).to be(descendent)
+      end
+    end
+  end
+
+  context 'when select_descendents is called' do
+    context 'with no children' do
+      subject { described_class.new('root', 0, 0) }
+      it 'returns an empty array' do
+        expect(subject.select_descendents(%w[root foo])).to eq([])
+      end
+    end
+
+    context 'with no descendents of the given type(s)' do
+      subject do
+        described_class.new('root', 0, 0, nil, [
+          described_class.new('foo', 0, 0),
+          described_class.new('bar', 0, 0, nil, [
+            described_class.new('baz', 0, 0)
+          ])
+        ])
+      end
+
+      it 'returns an empty array' do
+        expect(subject.select_descendents(%w[root qux])).to eq([])
+      end
+    end
+
+    context 'with multiple descendents of the given type(s)' do
+      let(:descendent_1) { described_class.new('bar', 0, 0) }
+      let(:descendent_3) { described_class.new('bar', 20, 20) }
+      let(:descendent_2) { described_class.new('baz', 10, 10, nil, [descendent_3]) }
+
+      subject do
+        described_class.new('root', 0, 0, nil, [
+          described_class.new('foo', 0, 0, nil, [
+            descendent_1
+          ]),
+          descendent_2,
+        ])
+      end
+
+      it 'returns all relevant descendents' do
+        expect(subject.select_descendents(%w[bar baz])).to match_array([descendent_1, descendent_2, descendent_3])
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-11-07 00:00:00.000000000 Z
+date: 2022-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: exifr
@@ -39,20 +39,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.2
-- !ruby/object:Gem::Dependency
-  name: ks
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: measurometer
   requirement: !ruby/object:Gem::Requirement
@@ -209,6 +195,8 @@ files:
 - lib/parsers/arw_parser.rb
 - lib/parsers/bmp_parser.rb
 - lib/parsers/cr2_parser.rb
+- lib/parsers/cr3_parser.rb
+- lib/parsers/cr3_parser/decoder.rb
 - lib/parsers/dpx_parser.rb
 - lib/parsers/dpx_parser/dpx_structs.rb
 - lib/parsers/exif_parser.rb
@@ -216,6 +204,7 @@ files:
 - lib/parsers/flac_parser.rb
 - lib/parsers/gif_parser.rb
 - lib/parsers/heif_parser.rb
+- lib/parsers/iso_base_media_file_format/decoder.rb
 - lib/parsers/jpeg_parser.rb
 - lib/parsers/m3u_parser.rb
 - lib/parsers/moov_parser.rb
@@ -257,12 +246,14 @@ files:
 - spec/parsers/arw_parser_spec.rb
 - spec/parsers/bmp_parser_spec.rb
 - spec/parsers/cr2_parser_spec.rb
+- spec/parsers/cr3_parser_spec.rb
 - spec/parsers/dpx_parser_spec.rb
 - spec/parsers/exif_parser_spec.rb
 - spec/parsers/fdx_parser_spec.rb
 - spec/parsers/flac_parser_spec.rb
 - spec/parsers/gif_parser_spec.rb
 - spec/parsers/heif_parser_spec.rb
+- spec/parsers/iso_base_media_file_format/decoder_spec.rb
 - spec/parsers/jpeg_parser_spec.rb
 - spec/parsers/m3u_parser_spec.rb
 - spec/parsers/moov_parser_spec.rb
@@ -295,7 +286,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="