format_parser 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ca1971c0e4a5b658e7d32101f068402fa7593fe7
-  data.tar.gz: '08494169e521c31c2620cce195b359eac0695cdc'
+  metadata.gz: 33fd6ee96653abe4059457cf1ba86113fc2c2a88
+  data.tar.gz: 7f042ecb080e9d04a78d7d22102dc3905f40b0d3
 SHA512:
-  metadata.gz: 6d969293a2bc611dc5cba25be05328006938727f9d766d74baea6d1278fd0d0e8b41a045e8946773c9a9596997ab57459bcc7ee18aa6058d5e4947706d740452
-  data.tar.gz: 97f2968333b20bdbda8231f188649236d3209a64a4a9a847e2376b2fb7d0122b906d6a57b2cb10fcbf7b46832f160cb25178f42b5fc68eac318145d9731be84c
+  metadata.gz: 9326f30de6b5344b9a17bd0832381ef5775094f1578173d5f5735b21aaff9f7358e7397ccf83b0c67ab7f4f71a7093ecf98d32007200f690026f953562211180
+  data.tar.gz: 393643805c0a9d995a07d8f7fdbd7afae28b2d617ccd0f05d456799b547a0f76fc8e160ec8e8127f20f39cd8b45ca4cd25dd516d64c8e3852a009f4e7c4ccda6

data/README.md

@@ -9,7 +9,7 @@ and [dimensions,](https://github.com/sstephenson/dimensions) borrowing from them
 
 ## Currently supported filetypes:
 
-`TIFF, PSD, PNG, MP3, JPEG, GIF, DPX, AIFF, WAV, FDX`
+`TIFF, PSD, PNG, MP3, JPEG, GIF, DPX, AIFF, WAV, FDX, MOV, MP4`
 
 ...with more on the way!
 
@@ -66,5 +66,12 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
 - c_39064__alienbomb__atmo-truck.wav is from [freesound](https://freesound.org/people/alienbomb/sounds/39064/) and is CC0 licensed
 - c_M1F1-Alaw-AFsp.wav and d_6_Channel_ID.wav are from a [McGill Engineering site](http://www-mmsp.ece.mcgill.ca/Documents/AudioFormats/WAVE/Samples.html)
 
-# FDX
+### MP3
+- Cassy.mp3 has been produced by WeTransfer and may be used with the library for the purposes of testing
+
+### FDX
 - fixture.fdx was created by one of the project maintainers and is MIT licensed
+
+### MOOV
+- bmff.mp4 is borrowed from the [bmff](https://github.com/zuku/bmff) project
+- Test_Circular MOV files were created by one of the project maintainers and are MIT licensed

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.1.5'
+  VERSION = '0.1.6'
 end

data/lib/parsers/moov_parser.rb

@@ -0,0 +1,80 @@
+class FormatParser::MOOVParser
+  include FormatParser::IOUtils
+
+  require_relative 'moov_parser/decoder'
+
+  # Maps values of the "ftyp" atom to something
+  # we can reasonably call "file type" (something
+  # usable as a filename extension)
+  FTYP_MAP = {
+    "qt  " => :mov,
+    "mp4 " => :mp4,
+    "m4a " => :m4a,
+  }
+
+  # It is currently not documented and not particularly well-tested,
+  # so not considered a public API for now
+  private_constant :Decoder 
+
+  def information_from_io(io)
+    return nil unless matches_moov_definition?(io)
+
+    # Now we know we are in a MOOV, so go back and parse out the atom structure.
+    # Parsing out the atoms does not read their contents - at least it doesn't
+    # for the atoms we consider opaque (one of which is the "mdat" atom which
+    # will be the prevalent part of the file body). We do not parse these huge
+    # atoms - we skip over them and note where they are located.
+    io.seek(0)
+
+    # We have to tell the parser how far we are willing to go within the stream.
+    # Knowing that we will bail out early anyway we will permit a large read. The
+    # branch parse calls will know the maximum size to read from the parent atom
+    # size that gets parsed just before.
+    max_read_offset = 0xFFFFFFFF
+    decoder = Decoder.new
+    atom_tree = decoder.extract_atom_stream(io, max_read_offset)
+
+    ftyp_atom = decoder.find_first_atom_by_path(atom_tree, 'ftyp')
+    file_type = ftyp_atom.field_value(:major_brand)
+
+    width, height = nil, nil
+
+    # Try to find the width and height in the tkhd
+    if tkhd = decoder.find_first_atom_by_path(atom_tree, 'moov', 'trak', 'tkhd')
+      width = tkhd.field_value(:track_width).first
+      height = tkhd.field_value(:track_height).first
+    end
+
+    # Try to find the "topmost" duration (respecting edits)
+    if mdhd = decoder.find_first_atom_by_path(atom_tree, 'moov', 'mvhd')
+      timescale, duration = mdhd.field_value(:tscale), mdhd.field_value(:duration)
+      media_duration_s = duration / timescale.to_f
+    end
+
+    FormatParser::FileInformation.new(
+      file_nature: :video,
+      file_type: file_type_from_moov_type(file_type),
+      width_px: width,
+      height_px: height,
+      media_duration_seconds: media_duration_s,
+      intrinsics: atom_tree,
+    )
+  end
+
+  private
+
+  def file_type_from_moov_type(file_type)
+    FTYP_MAP.fetch(file_type, :mov)
+  end
+
+  # An MPEG4/MOV/M4A will start with the "ftyp" atom. The atom must have a length
+  # of at least 8 (to accomodate the atom size and the atom type itself) plus the major
+  # and minor version fields. If we cannot find it we can be certain this is not our file.
+  def matches_moov_definition?(io)
+    maybe_atom_size, maybe_ftyp_atom_signature = safe_read(io, 8).unpack('N1a4')
+    minimum_ftyp_atom_size = 4 + 4 + 4 + 4
+    maybe_atom_size >= minimum_ftyp_atom_size && maybe_ftyp_atom_signature == 'ftyp'
+  end
+
+  FormatParser.register_parser_constructor self
+end

data/lib/parsers/moov_parser/decoder.rb

@@ -0,0 +1,285 @@
+# Handles decoding of MOV/MPEG4 atoms/boxes in a stream. Will recursively
+# read atoms and parse their data fields if applicable. Also contains
+# a few utility functions for finding atoms in a list etc.
+class FormatParser::MOOVParser::Decoder
+
+  class Atom < Struct.new(:at, :atom_size, :atom_type, :path, :children, :atom_fields)
+    def to_s
+      "%s (%s): %d bytes at offset %d" % [atom_type, path.join('.'), atom_size, at]
+    end
+
+    def field_value(data_field)
+      (atom_fields || {}).fetch(data_field)
+    end
+
+    def as_json(*a)
+      members.each_with_object({}) do |member_name, o|
+        o[member_name] = public_send(member_name).as_json(*a)
+      end
+    end
+  end
+
+  # Atoms (boxes) that are known to only contain children, no data fields
+  KNOWN_BRANCH_ATOM_TYPES = %w( moov mdia trak clip edts minf dinf stbl udta meta)
+
+  # Atoms (boxes) that are known to contain both leaves and data fields
+  KNOWN_BRANCH_AND_LEAF_ATOM_TYPES = %w( meta ) # the udta.meta thing used by iTunes
+
+  # Limit how many atoms we scan in sequence, to prevent derailments
+  MAX_ATOMS_AT_LEVEL = 128
+
+  # Finds the first atom in the given Array of Atom structs that
+  # matches the type, drilling down if a list of atom names is given
+  def find_first_atom_by_path(atoms, *atom_types)
+    type_to_find = atom_types.shift
+    requisite = atoms.find {|e| e.atom_type == type_to_find }
+
+    # Return if we found our match
+    return requisite if atom_types.empty?
+
+    # Return nil if we didn't find the match at this nesting level
+    return nil unless requisite
+
+    # ...otherwise drill further down
+    find_first_atom_by_path(requisite.children || [], *atom_types)
+  end
+
+  def parse_ftyp_atom(io, atom_size)
+    # Subtract 8 for the atom_size+atom_type,
+    # and 8 once more for the major_brand and minor_version. The remaining
+    # numbr of bytes is reserved for the compatible brands, 4 bytes per
+    # brand.
+    num_brands = (atom_size - 8 - 8) / 4
+    ret = {
+      major_brand: read_bytes(io, 4),
+      minor_version: read_binary_coded_decimal(io),
+      compatible_brands: (1..num_brands).map { read_bytes(io, 4) },
+    }
+  end
+
+  def parse_tkhd_atom(io, _)
+    version = read_byte_value(io)
+    is_v1 = version == 1
+    tkhd_info_bites = [
+      :version, version,
+      :flags, read_chars(io, 3),
+      :ctime, is_v1 ? read_64bit_uint(io) : read_32bit_uint(io),
+      :mtime, is_v1 ? read_64bit_uint(io) : read_32bit_uint(io),
+      :trak_id, read_32bit_uint(io),
+      :reserved_1, read_chars(io, 4),
+      :duration, is_v1 ? read_64bit_uint(io) : read_32bit_uint(io),
+      :reserved_2, read_chars(io, 8),
+      :layer, read_16bit_uint(io),
+      :alternate_group, read_16bit_uint(io),
+      :volume, read_16bit_uint(io),
+      :reserved_3, read_chars(io, 2),
+      :matrix_structure, (1..9).map { read_32bit_fixed_point(io) },
+      :track_width, read_32bit_fixed_point(io),
+      :track_height, read_32bit_fixed_point(io),
+    ]
+    repack(tkhd_info_bites)
+  end
+
+  def parse_mdhd_atom(io, _)
+    version = read_byte_value(io)
+    is_v1 = version == 1
+    mdhd_info_bites = [
+      :version, version,
+      :flags, read_bytes(io, 3),
+      :ctime, is_v1 ? read_64bit_uint(io) : read_32bit_uint(io),
+      :mtime, is_v1 ? read_64bit_uint(io) : read_32bit_uint(io),
+      :tscale, read_32bit_uint(io),
+      :duration, is_v1 ? read_64bit_uint(io) : read_32bit_uint(io),
+      :language, read_32bit_uint(io),
+      :quality, read_32bit_uint(io),
+    ]
+    repack(mdhd_info_bites)
+  end
+
+  def parse_vmhd_atom(io, _)
+    vmhd_info_bites = [
+      :version, read_byte_value(io),
+      :flags, read_bytes(io, 3),
+      :graphics_mode, read_bytes(io, 2),
+      :opcolor_r, read_32bit_uint(io),
+      :opcolor_g, read_32bit_uint(io),
+      :opcolor_b, read_32bit_uint(io),
+    ]
+    repack(vmhd_info_bites)
+  end
+
+  def parse_mvhd_atom(io, _)
+    version = read_byte_value(io)
+    is_v1 = version == 1
+    mvhd_info_bites = [
+      :version, version,
+      :flags, read_bytes(io, 3),
+      :ctime, is_v1 ? read_64bit_uint(io) : read_32bit_uint(io),
+      :mtime, is_v1 ? read_64bit_uint(io) : read_32bit_uint(io),
+      :tscale, read_32bit_uint(io),
+      :duration, is_v1 ? read_64bit_uint(io) : read_32bit_uint(io),
+      :preferred_rate, read_32bit_uint(io),
+      :reserved, read_bytes(io, 10),
+      :matrix_structure, (1..9).map { read_32bit_fixed_point(io) },
+      :preview_time, read_32bit_uint(io),
+      :preview_duration, read_32bit_uint(io),
+      :poster_time, read_32bit_uint(io),
+      :selection_time, read_32bit_uint(io),
+      :selection_duration, read_32bit_uint(io),
+      :current_time, read_32bit_uint(io),
+      :next_trak_id, read_32bit_uint(io),
+    ]
+    repack(mvhd_info_bites)
+  end
+
+  def parse_dref_atom(io, _)
+    dref_info_bites = [
+      :version, read_byte_value(io),
+      :flags, read_bytes(io, 3),
+      :num_entries, read_32bit_uint(io),
+    ]
+    dict = repack(dref_info_bites)
+    num_entries = dict[:num_entries]
+    entries = (1..num_entries).map do
+      dref_entry_bites = [
+        :size, read_32bit_uint(io),
+        :type, read_bytes(io, 4),
+        :version, read_bytes(io, 1),
+        :flags, read_bytes(io, 3),
+      ]
+      entry = repack(dref_entry_bites)
+      entry[:data] = read_bytes(io, entry[:size] - 12)
+      entry
+    end
+    dict[:entries] = entries
+    dict
+  end
+
+  def parse_elst_atom(io, _)
+    elst_info_bites = [
+      :version, read_byte_value(io),
+      :flags, read_bytes(io, 3),
+      :num_entries, read_32bit_uint(io),
+    ]
+    dict = repack(elst_info_bites)
+    is_v1 = dict[:version] == 1 # Usual is 0, version 1 has 64bit durations
+    num_entries = dict[:num_entries]
+    entries = (1..num_entries).map do
+      entry_bites = [
+        :track_duration, is_v1 ? read_64bit_uint(io) : read_32bit_uint(io),
+        :media_time, is_v1 ? read_64bit_uint(io) : read_32bit_uint(io),
+        :media_rate, read_32bit_uint(io),
+      ]
+      repack(entry_bites)
+    end
+    dict[:entries] = entries
+    dict
+  end
+
+  def parse_hdlr_atom(io, atom_size)
+    sub_io = StringIO.new(io.read(atom_size - 8))
+    hdlr_info_bites = [
+      :version, read_byte_value(sub_io),
+      :flags, read_bytes(sub_io, 3),
+      :component_type, read_bytes(sub_io, 4),
+      :component_subtype, read_bytes(sub_io, 4),
+      :component_manufacturer, read_bytes(sub_io, 4),
+      :component_flags, read_bytes(sub_io, 4),
+      :component_flags_mask, read_bytes(sub_io, 4),
+      :component_name, sub_io.read,
+    ]
+    repack(hdlr_info_bites)
+  end
+
+  def parse_atom_fields_per_type(io, atom_size, atom_type)
+    if respond_to?("parse_#{atom_type}_atom", including_privates = true)
+      send("parse_#{atom_type}_atom", io, atom_size)
+    else
+      nil # We can't look inside this leaf atom
+    end
+  end
+
+  # Recursive descent parser - will drill down to atoms which
+  # we know are permitted to have leaf/branch atoms within itself,
+  # and will attempt to recover the data fields for leaf atoms
+  def extract_atom_stream(io, max_read, current_branch = [])
+    initial_pos = io.pos
+    atoms = []
+    MAX_ATOMS_AT_LEVEL.times do
+      atom_pos = io.pos
+
+      if atom_pos - initial_pos >= max_read
+        break
+      end
+
+      size_and_type = io.read(4+4)
+      if size_and_type.to_s.bytesize < 8
+        break
+      end
+
+      atom_size, atom_type = size_and_type.unpack('Na4')
+
+      # If atom_size is specified to be 1, it is larger than what fits into the
+      # 4 bytes and we need to read it right after the atom type
+      if atom_size == 1
+        atom_size = read_64bit_uint(io)
+      end
+
+      children, fields = if KNOWN_BRANCH_AND_LEAF_ATOM_TYPES.include?(atom_type)
+        parse_atom_children_and_data_fields(io, atom_size, atom_type)
+      elsif KNOWN_BRANCH_ATOM_TYPES.include?(atom_type)
+        [extract_atom_stream(io, atom_size - 8, current_branch + [atom_type]), nil]
+      else # Assume leaf atom
+        [nil, parse_atom_fields_per_type(io, atom_size, atom_type)]
+      end
+
+      atoms << Atom.new(atom_pos, atom_size, atom_type, current_branch + [atom_type], children, fields)
+
+      io.seek(atom_pos + atom_size)
+    end
+    atoms
+  end
+
+  def read_16bit_fixed_point(io)
+    whole, fraction = io.read(2).unpack('CC')
+  end
+
+  def read_32bit_fixed_point(io)
+    whole, fraction = io.read(4).unpack('nn')
+  end
+
+  def read_chars(io, n)
+    io.read(n)
+  end
+
+  def read_byte_value(io)
+    io.read(1).unpack('C').first
+  end
+
+  def read_bytes(io, n)
+    io.read(n)
+  end
+
+  def read_16bit_uint(io)
+    io.read(2).unpack('n').first
+  end
+
+  def read_32bit_uint(io)
+    io.read(4).unpack('N').first
+  end
+
+  def read_64bit_uint(io)
+    io.read(8).unpack('Q>').first
+  end
+
+  def read_binary_coded_decimal(io)
+    bcd_string = io.read(4)
+    bcd_string.insert(0, '0') if bcd_string.length.odd?
+    [bcd_string].pack('H*').unpack('C*')
+  end
+
+  def repack(properties_to_packspecs)
+    keys, bytes = properties_to_packspecs.partition.with_index { |_, i| i.even? }
+    Hash[keys.zip(bytes)]
+  end
+end

data/lib/parsers/mp3_parser/id3_v2.rb

@@ -2,7 +2,6 @@ module FormatParser::MP3Parser::ID3V2
   def attempt_id3_v2_extraction(io)
     io.seek(0) # Only support header ID3v2
     header_bytes = io.read(10)
-
     return nil unless header_bytes
 
     header = parse_id3_v2_header(header_bytes)
@@ -50,12 +49,11 @@ module FormatParser::MP3Parser::ID3V2
   end
 
   def parse_id3_v2_frame(io)
-    id, size, flags = io.read(10).unpack('a4a4a2')
-    size = decode_syncsafe_int(size)
+    id, syncsafe_size, flags = io.read(10).unpack('a4a4a2')
+    size = decode_syncsafe_int(syncsafe_size)
     content = io.read(size)
-    if content.bytesize != size
-      raise "Expected to read #{size} bytes for ID3V2 frame #{id}, but got #{content.bytesize}"
-    end
+    # It might so happen in sutations of terrible invalidity that we end up
+    # with less data than advertised by the syncsafe size. We will just truck on.
     {id: id, size: size, flags: flags, content: content}
   end
 

data/spec/care_spec.rb

@@ -71,9 +71,12 @@ describe Care do
 
       subject = Care::IOWrapper.new(io_double, cache_double)
 
+      expect(subject.pos).to eq(0)
       subject.read(2)
       subject.read(3)
+      expect(subject.pos).to eq(5)
       subject.seek(11)
+      expect(subject.pos).to eq(11)
       subject.read(5)
 
       expect(cache_double.recorded_calls).to be_kind_of(Array)

data/spec/parsers/moov_parser_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+
+describe FormatParser::MOOVParser do
+
+  def deep_print_atoms(atoms, output, swimlanes = [])
+    return unless atoms
+
+    mid = '├'
+    last = '└'
+    horz = '─'
+    vert = '│'
+    cdn = '┬'
+    n_atoms = atoms.length
+  
+    atoms.each_with_index do |atom, i|
+      is_last_child = i == (n_atoms - 1)
+      has_children = atom.children && atom.children.any?
+      connector = is_last_child ? last : mid
+      connector_down = has_children ? cdn : horz
+      connector_left = is_last_child ? ' ' : vert
+
+      output << swimlanes.join << connector << connector_down << horz << atom.to_s << "\n"
+      if af = atom.atom_fields
+        af.each do |(field, value)|
+          output << swimlanes.join << connector_left << ('   %s: %s' % [field, value.inspect]) << "\n"
+        end
+      end
+      deep_print_atoms(atom.children, output, swimlanes + [connector_left])
+    end
+  end
+
+  Dir.glob(fixtures_dir + '/MOOV/**/*.*').sort.each do |moov_path|
+    it "is able to parse #{File.basename(moov_path)}" do
+      result = subject.information_from_io(File.open(moov_path, 'rb'))
+
+      expect(result).not_to be_nil
+      expect(result.file_nature).to eq(:video)
+      expect(result.width_px).to be > 0
+      expect(result.height_px).to be > 0
+      expect(result.media_duration_seconds).to be_kind_of(Float)
+      expect(result.media_duration_seconds).to be > 0
+
+      expect(result.intrinsics).not_to be_nil
+    end
+  end
+
+  it 'parses an M4A file and provides the necessary metadata'
+
+  it 'parses a MOV file and provides the necessary metadata' do
+    mov_path = fixtures_dir + '/MOOV/MOV/Test_Circular_ProRes422.mov'
+
+    result = subject.information_from_io(File.open(mov_path, 'rb'))
+
+    expect(result).not_to be_nil
+    expect(result.file_nature).to eq(:video)
+    expect(result.file_type).to eq(:mov)
+    expect(result.width_px).to eq(1920)
+    expect(result.height_px).to eq(1080)
+  end
+
+  it 'parses an MP4 video file and provides the necessary metadata' do
+    mov_path = fixtures_dir + '/MOOV/MP4/bmff.mp4'
+
+    result = subject.information_from_io(File.open(mov_path, 'rb'))
+
+    expect(result).not_to be_nil
+    expect(result.file_nature).to eq(:video)
+    expect(result.file_type).to eq(:mov)
+    expect(result.width_px).to eq(160)
+    expect(result.height_px).to eq(90)
+  end
+end

data/spec/parsers/mp3_parser_spec.rb

@@ -28,4 +28,20 @@ describe FormatParser::MP3Parser do
     expect(parsed.intrinsics).not_to be_nil
     expect(parsed.media_duration_seconds).to be_within(0.1).of(0.81)
   end
+
+  it 'parses the Cassy MP3' do
+    fpath = fixtures_dir + '/MP3/Cassy.mp3'
+    parsed = subject.information_from_io(File.open(fpath, 'rb'))
+
+    expect(parsed).not_to be_nil
+
+    expect(parsed.file_nature).to eq(:audio)
+    expect(parsed.file_type).to eq(:mp3)
+    expect(parsed.num_audio_channels).to eq(2)
+    expect(parsed.audio_sample_rate_hz).to eq(44100)
+    expect(parsed.intrinsics).not_to be_nil
+    expect(parsed.media_duration_seconds).to be_within(0.1).of(1102.46)
+
+    expect(parsed.intrinsics).not_to be_nil
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-14 00:00:00.000000000 Z
+date: 2018-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -156,6 +156,8 @@ files:
 - lib/parsers/fdx_parser.rb
 - lib/parsers/gif_parser.rb
 - lib/parsers/jpeg_parser.rb
+- lib/parsers/moov_parser.rb
+- lib/parsers/moov_parser/decoder.rb
 - lib/parsers/mp3_parser.rb
 - lib/parsers/mp3_parser/id3_v1.rb
 - lib/parsers/mp3_parser/id3_v2.rb
@@ -175,6 +177,7 @@ files:
 - spec/parsers/fdx_parser_spec.rb
 - spec/parsers/gif_parser_spec.rb
 - spec/parsers/jpeg_parser_spec.rb
+- spec/parsers/moov_parser_spec.rb
 - spec/parsers/mp3_parser_spec.rb
 - spec/parsers/png_parser_spec.rb
 - spec/parsers/psd_parser_spec.rb
@@ -205,7 +208,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: A library for efficient parsing of file metadata