format_parser 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 205c4099f44c0080b53e210ea18cebd4484476ba
-  data.tar.gz: d8145f8f77be44dab386585679a8c7f3ce48869d
+  metadata.gz: b0ef6923a01b8fbe52f4491979a3be1224ea7018
+  data.tar.gz: a19859f36a81154f73d1071857834cd0e7e1ca74
 SHA512:
-  metadata.gz: ad845e5216be2e71e81205cade268d72b1be1ee2e5721a7877a5222a03a43c0f10d18abad8fc1c553165ade0665c7bba9ef5d5a79aa954e8c587ac0d027d6d89
-  data.tar.gz: 172c0d12205c778dadefe57010c5cfe6635d5398eea385048b8e41b3558a326cba4c332e37b9259abef4589d78f31c39bffb7b9c6adb416417b771cdb2b49272
+  metadata.gz: ca6bd5d8324a4dcb41d6f3137a1ab621a016acf8e9fafae5983bb8c325f883ecc72b4046d774e423e5e4ca7f35d048dae20a1e6ca2287d842c111c2714e2d606
+  data.tar.gz: 99d6517341e8b48635c8540e7f92cb48e772ed02047eb6cd56412b21de2411c06d64d43878ffa1a3ce43bf8e82b7e886eef4f00b221bfdfe42bafc19435f3f35

data/README.md

@@ -7,6 +7,12 @@ minimum amount of data possible.
 `format_parser` is inspired by [imagesize,](https://rubygems.org/gem/imagesize) [fastimage](https://github.com/sdsykes/fastimage)
 and [dimensions,](https://github.com/sstephenson/dimensions) borrowing from them where appropriate.
 
+## Currently supported filetypes:
+
+`TIFF, PSD, PNG, MP3, JPEG, GIF, DPX, AIFF`
+
+...with more on the way!
+
 ## Basic usage
 
 Pass an IO object that responds to `read` and `seek` to `FormatParser`.

data/format_parser.gemspec

@@ -32,6 +32,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_dependency 'ks', '~> 0.0.1'
   spec.add_dependency 'exifr', '~> 1.0'
   spec.add_dependency 'faraday', '~> 0.13'
   

data/lib/care.rb

@@ -39,6 +39,10 @@ class Care
       clear
       @io.close if @io.respond_to?(:close)
     end
+
+    def size
+      @io.size
+    end
   end
 
   # Stores cached pages of data from the given IO as strings.
@@ -58,7 +62,10 @@ class Care
     # or fetch pages where necessary
     def byteslice(io, at, n_bytes)
       if n_bytes < 1
-        raise ArgumentError, "The number of bytes to fetch must be a positive Integer" 
+        raise ArgumentError, "The number of bytes to fetch must be a positive Integer"
+      end
+      if at < 0
+        raise ArgumentError, "Negative offsets are not supported (got #{at})"
       end
 
       first_page = at / @page_size
@@ -124,7 +131,7 @@ class Care
         # to read following this one, so we can also optimize
         @lowest_known_empty_page = page_i + 1
       end
-        
+
       read_result
     end
   end

data/lib/file_information.rb

@@ -58,6 +58,10 @@ module FormatParser
     # as an Integer
     attr_accessor :media_duration_frames
 
+    # If a parser wants to provide any extra information to the caller
+    # it can be placed here
+    attr_accessor :intrinsics
+
     # Only permits assignments via defined accessors
     def initialize(**attributes)
       attributes.map { |(k, v)| public_send("#{k}=", v) }

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.1.2'
+  VERSION = '0.1.3'
 end

data/lib/parsers/mp3_parser.rb

@@ -0,0 +1,246 @@
+require 'ks'
+
+class FormatParser::MP3Parser
+  require_relative 'mp3_parser/id3_v1'
+  require_relative 'mp3_parser/id3_v2'
+
+  class MPEGFrame < Ks.strict(:offset_in_file, :mpeg_id, :channels, :sample_rate, :frame_length, :frame_bitrate)
+  end
+
+  class VBRHeader < Ks.strict(:frames, :byte_count, :toc_entries, :vbr_scale)
+  end
+
+  class MP3Info < Ks.strict(:duration_seconds, :num_channels, :sampling_rate)
+  end
+
+  class InvalidDeepFetch < KeyError
+  end
+
+  # We limit the number of MPEG frames we scan
+  # to obtain our duration estimation
+  MAX_FRAMES_TO_SCAN = 128
+
+  # Default frame size for mp3
+  SAMPLES_PER_FRAME = 1152
+
+  def information_from_io(io)
+    # Read the last 128 bytes which might contain ID3v1
+    id3_v1 = ID3V1.attempt_id3_v1_extraction(io)
+    # Read the header bytes that might contain ID3v1
+    id3_v2 = ID3V2.attempt_id3_v2_extraction(io)
+
+    # Compute how many bytes are occupied by the actual MPEG frames
+    ignore_bytes_at_tail = id3_v1 ? 128 : 0
+    ignore_bytes_at_head = id3_v2 ? io.pos : 0
+    bytes_used_by_frames = io.size - ignore_bytes_at_tail - ignore_bytes_at_tail
+    
+    io.seek(ignore_bytes_at_head)
+
+    maybe_xing_header, initial_frames = parse_mpeg_frames(io)
+
+    return nil if initial_frames.empty?
+
+    first_frame = initial_frames.first
+
+    file_info = FormatParser::FileInformation.new(
+      file_nature: :audio,
+      file_type: :mp3,
+      num_audio_channels: first_frame.channels,
+      audio_sample_rate_hz: first_frame.sample_rate,
+      # media_duration_frames is omitted because the frames
+      # in MPEG are not the same thing as in a movie file - they
+      # do not tell anything of substance
+      intrinsics: {
+        id3_v1: id3_v1 ? id3_v1.to_h : nil,
+        id3_v2: id3_v2 ? id3_v2.map(&:to_h) : nil,
+        xing_header: maybe_xing_header.to_h,
+        initial_frames: initial_frames.map(&:to_h)
+      }
+    )
+
+    if maybe_xing_header
+      duration = maybe_xing_header.frames * SAMPLES_PER_FRAME / first_frame.sample_rate.to_f
+      bit_rate = maybe_xing_header.byte_count * 8 / duration / 1000
+      file_info.media_duration_seconds = duration
+      return file_info
+    end
+
+    # Estimate duration using the frames we did parse - to have an exact one
+    # we would need to have all the frames and thus read most of the file
+    avg_bitrate = float_average_over(initial_frames, :frame_bitrate)
+    avg_frame_size = float_average_over(initial_frames, :frame_length)
+    avg_sample_rate = float_average_over(initial_frames, :sample_rate)
+
+    est_frame_count = bytes_used_by_frames / avg_frame_size
+    est_samples = est_frame_count * SAMPLES_PER_FRAME
+    est_duration_seconds = est_samples / avg_sample_rate
+
+    file_info.media_duration_seconds = est_duration_seconds
+    return file_info
+  end
+
+  private
+  
+  # The implementation of the MPEG frames parsing is mostly based on tinytag,
+  # a sweet little Python library for parsing audio metadata - do check it out
+  # if you have a minute. https://pypi.python.org/pypi/tinytag
+  def parse_mpeg_frames(io)
+    mpeg_frames = []
+    
+    MAX_FRAMES_TO_SCAN.times do |frame_i|
+      # Read through until we can latch onto the 11 sync bits. Read in 4-byte
+      # increments to save on read() calls
+      data = io.read(4)
+
+      # If we are at EOF - stop iterating
+      break unless data && data.bytesize == 4
+
+      # Look for the sync pattern. It can be either the last byte being 0xFF,
+      # or any of the 2 bytes in sequence being 0xFF and > 0xF0.
+      four_bytes = data.unpack('C4')
+      seek_jmp = sync_bytes_offset_in_4_byte_seq(four_bytes)
+      if seek_jmp > 0
+        io.seek(io.pos + seek_jmp)
+        next
+      end
+
+      # Once we are past that stage we have latched onto a sync frame header
+      sync, conf, bitrate_freq, rest = four_bytes
+      frame_detail = parse_mpeg_frame_header(io.pos - 4, sync, conf, bitrate_freq, rest)
+      mpeg_frames << frame_detail
+
+      # There might be a xing header in the first frame that contains
+      # all the info we need, otherwise parse multiple frames to find the
+      # accurate average bitrate
+      if frame_i == 0
+        frame_data_str = io.read(frame_detail.frame_length)
+        io.seek(io.pos - frame_detail.frame_length)
+        xing_header = attempt_xing_header(frame_data_str)
+        if xing_header_usable_for_duration?(xing_header)
+          return [xing_header, mpeg_frames]
+        end
+      end
+      if frame_detail.frame_length > 1  # jump over current frame body
+        io.seek(io.pos + frame_detail.frame_length - 4)
+      end
+    end
+    [nil, mpeg_frames]
+  rescue InvalidDeepFetch # A frame was invalid - bail out since it's unlikely we can recover
+    [nil, mpeg_frames]
+  end
+
+  def parse_mpeg_frame_header(offset_in_file, sync, conf, bitrate_freq, rest)
+    # see this page for the magic values used in mp3:
+    # http:/www.mpgedit.org/mpgedit/mpeg_format/mpeghdr.htm
+    samplerates = [
+        [11025, 12000,  8000],  # MPEG 2.5
+        [],                     # reserved
+        [22050, 24000, 16000],  # MPEG 2
+        [44100, 48000, 32000],  # MPEG 1
+    ]
+    v1l1 = [0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448, 0]
+    v1l2 = [0, 32, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384, 0]
+    v1l3 = [0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0]
+    v2l1 = [0, 32, 48, 56, 64, 80, 96, 112, 128, 144, 160, 176, 192, 224, 256, 0]
+    v2l2 = [0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0]
+    v2l3 = v2l2
+    bitrate_by_version_by_layer = [
+        [nil, v2l3, v2l2, v2l1],  # MPEG Version 2.5  # note that the layers go
+        nil,                      # reserved          # from 3 to 1 by design.
+        [nil, v2l3, v2l2, v2l1],  # MPEG Version 2    # the first layer id is
+        [nil, v1l3, v1l2, v1l1],  # MPEG Version 1    # reserved
+    ]
+    samples_per_frame = 1152  # the default frame size for mp3
+    channels_per_channel_mode = [
+        2,  # 00 Stereo
+        2,  # 01 Joint stereo (Stereo)
+        2,  # 10 Dual channel (2 mono channels)
+        1,  # 11 Single channel (Mono)
+    ]
+    
+    br_id = (bitrate_freq >> 4) & 0x0F  # biterate id
+    sr_id = (bitrate_freq >> 2) & 0x03  # sample rate id
+    padding = bitrate_freq & 0x02 > 0 ? 1 : 0
+    mpeg_id = (conf >> 3) & 0x03
+    layer_id = (conf >> 1) & 0x03
+    channel_mode = (rest >> 6) & 0x03
+    channels = channels_per_channel_mode.fetch(channel_mode)
+    sample_rate = deep_fetch(samplerates, mpeg_id, sr_id)
+    frame_bitrate = deep_fetch(bitrate_by_version_by_layer, mpeg_id, layer_id, br_id)
+    frame_length = (144000 * frame_bitrate) / sample_rate + padding
+    MPEGFrame.new(
+      offset_in_file: offset_in_file,
+      mpeg_id: mpeg_id,
+      channels: channels,
+      sample_rate: sample_rate,
+      frame_length: frame_length,
+      frame_bitrate: frame_bitrate,
+    )
+  end
+
+  # Scan 4 byte values, and check whether there is
+  # a pattern of the 11 set bits anywhere within it
+  # or whether there is the 0xFF byte at the end
+  def sync_bytes_offset_in_4_byte_seq(four_bytes)
+    four_bytes[0...3].each_with_index do |byte, i|
+      next_byte = four_bytes[i+1]
+      if byte == 0xFF && next_byte > 0xE0
+        return i
+      end
+    end
+    four_bytes[-1] == 0xFF ? 3 : 4
+  end
+
+  def attempt_xing_header(frame_body)
+    unless xing_offset = frame_body.index("Xing")
+      return nil # No Xing in this frame
+    end
+
+    io = StringIO.new(frame_body)
+    io.seek(xing_offset + 4) # Include the length of "Xing" itself
+
+    # https://www.codeproject.com/Articles/8295/MPEG-Audio-Frame-Header#XINGHeader
+    header_flags, _ = io.read(4).unpack('s>s>')
+    frames = byte_count = toc = vbr_scale = nil
+    
+    if header_flags & 1  # FRAMES FLAG
+      frames = io.read(4).unpack('N1').first
+    end
+
+    if header_flags & 2  # BYTES FLAG
+      byte_count = io.read(4).unpack('N1').first
+    end
+
+    if header_flags & 4  # TOC FLAG
+      toc = io.read(100).unpack('C100')
+    end
+
+    if header_flags & 8  # VBR SCALE FLAG
+      vbr_scale = io.read(4).unpack('N1').first
+    end
+
+    VBRHeader.new(frames: frames, byte_count: byte_count, toc_entries: toc, vbr_scale: vbr_scale)
+  end
+
+  def average_bytes_and_bitrate(mpeg_frames)
+    avg_bytes_per_frame = initial_frames.map(&:frame_length).inject(&:+) / initial_frames.length.to_f
+    avg_bitrate_per_frame = initial_frames.map(&:frame_bitrate).inject(&:+) / initial_frames.length.to_f
+    [avg_bytes_per_frame, avg_bitrate_per_frame]
+  end
+  
+  def xing_header_usable_for_duration?(xing_header)
+    xing_header && xing_header.frames && xing_header.byte_count && xing_header.vbr_scale
+  end
+
+  def float_average_over(enum, property)
+    enum.map(&property).inject(&:+) / enum.length.to_f
+  end
+
+  def deep_fetch(from, *keys)
+    keys.inject(from) { |receiver, key_or_idx| receiver.fetch(key_or_idx) }
+  rescue KeyError, IndexError, NoMethodError
+    raise InvalidDeepFetch, "Could not retrieve #{keys.inspect} from #{from.inspect}"
+  end
+
+  FormatParser.register_parser_constructor self
+end

data/lib/parsers/mp3_parser/id3_v1.rb

@@ -0,0 +1,54 @@
+module FormatParser::MP3Parser::ID3V1
+  PACKSPEC = [
+    :tag, :a3,
+    :song_name, :a30,
+    :artist, :a30,
+    :album, :a30,
+    :year, :N1,
+    :comment, :a30,
+    :genre, :C,
+  ]
+  packspec_keys = PACKSPEC.select.with_index{|_, i| i.even? }
+  TAG_SIZE_BYTES = 128
+
+  class TagInformation < Struct.new(*packspec_keys)
+  end
+
+  def attempt_id3_v1_extraction(io)
+    if io.size < TAG_SIZE_BYTES # Won't fit the ID3v1 regardless
+      return nil
+    end
+
+    io.seek(io.size - 128)
+    trailer_bytes = io.read(128)
+
+    unless trailer_bytes && trailer_bytes.byteslice(0, 3) == 'TAG'
+      return nil
+    end
+
+    id3_v1 = parse_id3_v1(trailer_bytes)
+
+    # If all of the resulting strings are empty this ID3v1 tag is invalid and
+    # we should ignore it.
+    strings_from_id3v1 = id3_v1.values.select{|e| e.is_a?(String) && e != 'TAG' }
+    if strings_from_id3v1.all?(&:empty?)
+      return nil
+    end
+
+    id3_v1
+  end
+
+  def parse_id3_v1(byte_str)
+    keys, values = PACKSPEC.partition.with_index {|_, i| i.even? }
+    unpacked_values = byte_str.unpack(values.join)
+    unpacked_values.map! {|e| e.is_a?(String) ? trim_id3v1_string(e) : e }
+    TagInformation.new(unpacked_values)
+  end
+
+  # Remove trailing whitespace and trailing nullbytes
+  def trim_id3v1_string(str)
+    str.tr("\x00".b, '').strip
+  end
+
+  extend self
+end

data/lib/parsers/mp3_parser/id3_v2.rb

@@ -0,0 +1,86 @@
+module FormatParser::MP3Parser::ID3V2
+  def attempt_id3_v2_extraction(io)
+    io.seek(0) # Only support header ID3v2
+    header_bytes = io.read(10)
+
+    return nil unless header_bytes
+
+    header = parse_id3_v2_header(header_bytes)
+    return nil unless header[:tag] == 'ID3'
+    return nil unless header[:size] > 0
+
+    header_tag_payload = io.read(header[:size])
+    header_tag_payload = StringIO.new(header_tag_payload)
+
+    return nil unless header_tag_payload.size == header[:size]
+
+    frames = []
+    loop do
+      break if header_tag_payload.eof?
+      frame = parse_id3_v2_frame(header_tag_payload)
+      # Some files include padding, which is there so that when you edit ID3v2
+      # you do not have to overwrite the entire file - you can use this padding to
+      # add some more tags or to grow the existing ones. In practice if we hit
+      # something with a type of "0x00000000" we have entered the padding zone and
+      # there is no point in parsing further
+      if frame[:id] == "\x00\x00\x00\x00".b
+        break
+      else
+        frames << frame
+      end
+    end
+    frames
+  end
+
+  def parse_id3_v2_header(byte_str)
+    packspec = [
+      :tag, :a3,
+      :version, :a2,
+      :flags, :C1,
+      :size, :a4,
+    ]
+    keys, values = packspec.partition.with_index {|_, i| i.even? }
+    unpacked_values = byte_str.unpack(values.join)
+    header_data = Hash[keys.zip(unpacked_values)]
+    
+    header_data[:version] = header_data[:version].unpack('C2')
+    header_data[:size] = decode_syncsafe_int(header_data[:size])
+
+    header_data
+  end
+
+  def parse_id3_v2_frame(io)
+    id, size, flags = io.read(10).unpack('a4a4a2')
+    size = decode_syncsafe_int(size)
+    content = io.read(size)
+    if content.bytesize != size
+      raise "Expected to read #{size} bytes for ID3V2 frame #{id}, but got #{content.bytesize}"
+    end
+    {id: id, size: size, flags: flags, content: content}
+  end
+
+  # ID3v2 uses "unsynchronized integers", which are unsigned integers smeared
+  # over multiple bytes in such a manner that the first bit is always 0 (unset).
+  # This is done so that ID3v2 incompatible decoders will not by accident see
+  # the 0xFF0xFF0xFF0xFF sequence anywhere that can be mistaken for the MPEG frame
+  # synchronisation header. Effectively it is a 7 bit big-endian unsigned integer
+  # encoding.
+  #
+  # 8 bit 255 (0xFF) encoded in this mannner takes 16 bits instead,
+  # and looks like this: `0b00000001 01111111`. Note how it avoids having
+  # the first bit of the second byte be 1.
+  # This method decodes an unsigned integer packed in this fashion
+  def decode_syncsafe_int(bytes)
+    size = 0
+    j = 0
+    i = bytes.bytesize - 1
+    while i >= 0
+      size += 128**i * (bytes.getbyte(j) & 0x7f)
+      j += 1
+      i -= 1
+    end
+    size
+  end
+
+  extend self
+end

data/lib/read_limiter.rb

@@ -1,5 +1,6 @@
 class FormatParser::ReadLimiter
   NO_LIMIT = nil
+
   class BudgetExceeded < StandardError
   end
 
@@ -30,6 +31,10 @@ class FormatParser::ReadLimiter
     @io.seek(to_offset)
   end
 
+  def size
+    @io.size
+  end
+
   def read(n)
     @bytes += n
     @reads += 1

data/spec/care_spec.rb

@@ -14,6 +14,13 @@ describe Care do
       expect(cache.byteslice(source, 120, 12)).to be_nil
     end
 
+    it 'raises on a negative read offset' do
+      cache = Care::Cache.new(3)
+      expect {
+        cache.byteslice(source, -2, 3)
+      }.to raise_error(/negative/i)
+    end
+
     it 'can be cleared' do
       cache = Care::Cache.new(3)
       expect(cache.byteslice(source, 0, 3)).to eq("Hel")
@@ -80,7 +87,7 @@ describe Care do
       methods_not_covered = Set.new(FormatParser::IOConstraint.public_instance_methods) - Set.new(Care::IOWrapper.public_instance_methods)
       expect(methods_not_covered).to be_empty
     end
-    
+
     it 'forwards calls to size() to the underlying IO' do
       io_double = double('IO')
       expect(io_double).to receive(:size).and_return(123)

data/spec/parsers/mp3_parser_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe FormatParser::MP3Parser do
+  it 'decodes and estimates duration for a VBR MP3' do
+    fpath = fixtures_dir + '/MP3/atc_fixture_vbr.mp3'
+    parsed = subject.information_from_io(File.open(fpath, 'rb'))
+
+    expect(parsed).not_to be_nil
+
+    expect(parsed.file_nature).to eq(:audio)
+    expect(parsed.file_type).to eq(:mp3)
+    expect(parsed.num_audio_channels).to eq(2)
+    expect(parsed.audio_sample_rate_hz).to eq(44100)
+    expect(parsed.intrinsics).not_to be_nil
+    expect(parsed.media_duration_seconds).to be_within(0.1).of(0.836)
+  end
+
+  it 'decodes and estimates duration for a CBR MP3' do
+    fpath = fixtures_dir + '/MP3/atc_fixture_cbr.mp3'
+    parsed = subject.information_from_io(File.open(fpath, 'rb'))
+
+    expect(parsed).not_to be_nil
+
+    expect(parsed.file_nature).to eq(:audio)
+    expect(parsed.file_type).to eq(:mp3)
+    expect(parsed.num_audio_channels).to eq(2)
+    expect(parsed.audio_sample_rate_hz).to eq(44100)
+    expect(parsed.intrinsics).not_to be_nil
+    expect(parsed.media_duration_seconds).to be_within(0.1).of(0.81)
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Noah Berman
@@ -9,8 +9,22 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-08 00:00:00.000000000 Z
+date: 2018-01-09 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: ks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.1
 - !ruby/object:Gem::Dependency
   name: exifr
   requirement: !ruby/object:Gem::Requirement
@@ -141,6 +155,9 @@ files:
 - lib/parsers/exif_parser.rb
 - lib/parsers/gif_parser.rb
 - lib/parsers/jpeg_parser.rb
+- lib/parsers/mp3_parser.rb
+- lib/parsers/mp3_parser/id3_v1.rb
+- lib/parsers/mp3_parser/id3_v2.rb
 - lib/parsers/png_parser.rb
 - lib/parsers/psd_parser.rb
 - lib/parsers/tiff_parser.rb
@@ -155,6 +172,7 @@ files:
 - spec/parsers/exif_parser_spec.rb
 - spec/parsers/gif_parser_spec.rb
 - spec/parsers/jpeg_parser_spec.rb
+- spec/parsers/mp3_parser_spec.rb
 - spec/parsers/png_parser_spec.rb
 - spec/parsers/psd_parser_spec.rb
 - spec/parsers/tiff_parser_spec.rb