format_parser 0.12.4 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 03325dd0dc412571fd66d0c94226b9b88d7ae88b9bb6a985fe7c29f226c87b64
-  data.tar.gz: cbb1e33a3bffa36e832a064d9f28c6f8ebb2985fee397560d3ce19d971c36b96
+  metadata.gz: f2bd20132739f75daaae94963ac56837e4852af4bd891d0b74abfef85e110eae
+  data.tar.gz: 442e4d4a2cdbe5c5fa1fd46b12e218f33c1e52d4706d94d751800a12913aa6ae
 SHA512:
-  metadata.gz: 95c9d34a4469670ee69b718dac997fc3d1cc471071979ceb9df0010bff191d8b4c39e02f34669a13d831e40d918ad79889a6ff3b3ff0272df40b6160ee63bf8c
-  data.tar.gz: 417dbdcd4ea63939e060df8a8dd6e4927b4d9ec2de01cfab7d7b9ecf9701f56c5a93494cae7fb30361178d2b05b840f6c55928fffed73742829374acc8e24aff
+  metadata.gz: b51a7884bb770ec2bc4d49c7583d9215acf162258368ab557d2da2475083d9ba85e7c00c34506f8cc01c812051189b3cc3e30aaea8c919bc0b7166f255947b96
+  data.tar.gz: ba4a3377d265d6f9d1a5887bd6de7cdc83a92c39dbfcb04c97182be265514ffc3b2d36bad1938dc0385d264602fbd197c5e8c7321cc7ad097ddb2303a612d022

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.13.0
+* Replace the homegrown ID3 parser with [id3tag](https://github.com/krists/id3tag) - this introduces id3tag
+  as a dependency in addition to `exifr`, but the gains are substantial.
+
 ## 0.12.4
 * Ensure JPEG recognition only runs when the JPEG SOI marker is detected **at the start** of file. Previously
   the JPEG parser would scan for the marker, sometimes finding it (appropriately) in places like... MP3 album

data/format_parser.gemspec

@@ -32,6 +32,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'ks', '~> 0.0.1'
   spec.add_dependency 'exifr', '~> 1.0'
+  spec.add_dependency 'id3tag', '~> 0.10'
   spec.add_dependency 'faraday', '~> 0.13'
 
   spec.add_development_dependency 'rspec', '~> 3.0'

data/lib/attributes_json.rb

@@ -11,7 +11,6 @@
 #   the_foo.number_of_bars = 42
 #   the_foo.as_json #=> {:number_of_bars => 42}
 module FormatParser::AttributesJSON
-  UNICODE_REPLACEMENT_CHAR = [0xFFFD].pack('U')
   MAXIMUM_JSON_NESTING_WHEN_SANITIZING = 256
 
   # Implements a sane default `as_json` for an object
@@ -49,7 +48,7 @@ module FormatParser::AttributesJSON
     when Float::INFINITY
       nil
     when String
-      value.encode(Encoding::UTF_8, undef: :replace, replace: UNICODE_REPLACEMENT_CHAR)
+      FormatParser.string_to_lossy_utf8(value)
     when Hash
       Hash[value.map { |k, v| [_sanitize_json_value(k, nesting + 1), _sanitize_json_value(v, nesting + 1)] }]
     when Array

data/lib/format_parser.rb

@@ -243,6 +243,11 @@ module FormatParser
     end
   end
 
+  def self.string_to_lossy_utf8(str)
+    replacement_char = [0xFFFD].pack('U')
+    str.encode(Encoding::UTF_8, undef: :replace, replace: replacement_char)
+  end
+
   Dir.glob(__dir__ + '/parsers/*.rb').sort.each do |parser_file|
     require parser_file
   end

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.12.4'
+  VERSION = '0.13.0'
 end

data/lib/parsers/mp3_parser.rb

@@ -1,8 +1,8 @@
 require 'ks'
+require 'id3tag'
 
 class FormatParser::MP3Parser
-  require_relative 'mp3_parser/id3_v1'
-  require_relative 'mp3_parser/id3_v2'
+  require_relative 'mp3_parser/id3_extraction'
 
   class MPEGFrame < Ks.strict(:offset_in_file, :mpeg_id, :channels, :sample_rate, :frame_length, :frame_bitrate)
   end
@@ -26,21 +26,50 @@ class FormatParser::MP3Parser
   # For some edge cases
   ZIP_LOCAL_ENTRY_SIGNATURE = "PK\x03\x04\x14\x00".b
 
-  def call(io)
+  # Wraps the Tag object returned by ID3Tag in such
+  # a way that a usable JSON representation gets
+  # returned
+  class TagWrapper < SimpleDelegator
+    include FormatParser::AttributesJSON
+
+    MEMBERS = [:artist, :title, :album, :year, :track_nr, :genre, :comments, :unsychronized_transcription]
+
+    def self.new(wrapped)
+      wrapped ? super : nil
+    end
+
+    def to_h
+      tag = __getobj__
+      MEMBERS.each_with_object({}) do |k, h|
+        # ID3Tag sometimes raises when trying to find an unknown genre.
+        # If this guard is removed, it fails when trying to do a gsub on a nil,
+        # in /lib/id3tag/frames/v2/genre_frame/genre_parser_pre_24.rb:25:in `just_genres'
+        value = tag.public_send(k) rescue nil
+        h[k] = value if value
+      end
+    end
+
+    def as_json(*)
+      to_h
+    end
+  end
+
+  def call(raw_io)
+    io = FormatParser::IOConstraint.new(raw_io)
+
     # Special case: some ZIPs (Office documents) did detect as MP3s.
     # To avoid having that happen, we check for the PKZIP signature -
     # local entry header signature - at the very start of the file
     return if io.read(6) == ZIP_LOCAL_ENTRY_SIGNATURE
     io.seek(0)
 
-    # Read the last 128 bytes which might contain ID3v1
-    id3_v1 = ID3V1.attempt_id3_v1_extraction(io)
-    # Read the header bytes that might contain ID3v1
-    id3_v2 = ID3V2.attempt_id3_v2_extraction(io)
+    # Read all the ID3 tags (or at least attempt to)
+    id3v1 = ID3Extraction.attempt_id3_v1_extraction(io)
+    tags = [id3v1, ID3Extraction.attempt_id3_v2_extraction(io)].compact
 
     # Compute how many bytes are occupied by the actual MPEG frames
-    ignore_bytes_at_tail = id3_v1 ? 128 : 0
-    ignore_bytes_at_head = id3_v2 ? io.pos : 0
+    ignore_bytes_at_tail = id3v1 ? 128 : 0
+    ignore_bytes_at_head = io.pos
     bytes_used_by_frames = io.size - ignore_bytes_at_tail - ignore_bytes_at_tail
 
     io.seek(ignore_bytes_at_head)
@@ -53,17 +82,12 @@ class FormatParser::MP3Parser
 
     file_info = FormatParser::Audio.new(
       format: :mp3,
-      num_audio_channels: first_frame.channels,
-      audio_sample_rate_hz: first_frame.sample_rate,
       # media_duration_frames is omitted because the frames
       # in MPEG are not the same thing as in a movie file - they
       # do not tell anything of substance
-      intrinsics: {
-        id3_v1: id3_v1 ? id3_v1.to_h : nil,
-        id3_v2: id3_v2 ? id3_v2.map(&:to_h) : nil,
-        xing_header: maybe_xing_header.to_h,
-        initial_frames: initial_frames.map(&:to_h)
-      }
+      num_audio_channels: first_frame.channels,
+      audio_sample_rate_hz: first_frame.sample_rate,
+      intrinsics: blend_id3_tags_into_hash(*tags).merge(id3tags: tags)
     )
 
     if maybe_xing_header
@@ -244,5 +268,11 @@ class FormatParser::MP3Parser
     raise InvalidDeepFetch, "Could not retrieve #{keys.inspect} from #{from.inspect}"
   end
 
+  def blend_id3_tags_into_hash(*tags)
+    tags.each_with_object({}) do |tag, h|
+      h.merge!(TagWrapper.new(tag).to_h)
+    end
+  end
+
   FormatParser.register_parser self, natures: :audio, formats: :mp3
 end

data/lib/parsers/mp3_parser/id3_extraction.rb

@@ -0,0 +1,76 @@
+module FormatParser::MP3Parser::ID3Extraction
+  ID3V1_TAG_SIZE_BYTES = 128
+  ID3V2_TAG_VERSIONS = ["\x43\x00".b, "\x03\x00".b, "\x02\x00".b]
+  MAX_SIZE_FOR_ID3V2 = 1 * 1024 * 1024
+
+  extend FormatParser::IOUtils
+
+  def attempt_id3_v1_extraction(io)
+    return if io.size < ID3V1_TAG_SIZE_BYTES # Won't fit the ID3v1 regardless
+
+    io.seek(io.size - 128)
+    trailer_bytes = io.read(128)
+
+    return unless trailer_bytes && trailer_bytes.bytesize == ID3V1_TAG_SIZE_BYTES
+    return unless trailer_bytes.byteslice(0, 3) == 'TAG'
+
+    buf = StringIO.new(trailer_bytes)
+    swallow_exceptions { ID3Tag.read(buf, :v1) }
+  end
+
+  def attempt_id3_v2_extraction(io)
+    io.seek(0) # Only support header ID3v2
+    header = parse_id3_v2_header(io)
+    return unless header[:tag] == 'ID3' && header[:size] > 0
+    return unless ID3V2_TAG_VERSIONS.include?(header[:version])
+
+    id3_tag_size = io.pos + header[:size]
+
+    # Here we got to pay attention. The tag size encoded in
+    # the ID3 header is a 4-byte unsigned int. Meaning it
+    # can hold values up to 256 MB. We do not want to read
+    # that much since we are pulling that data into memory -
+    # and it would also make the parser easily exploitable.
+    # We will set a "hard" limit beyound which we will simply
+    # refuse to read those tags at all.
+    if id3_tag_size > MAX_SIZE_FOR_ID3V2
+      io.seek(id3_tag_size) # For reading the frames
+      return
+    end
+
+    io.seek(0)
+    blob = safe_read(io, id3_tag_size)
+
+    swallow_exceptions { ID3Tag.read(StringIO.new(blob), :v2) }
+  rescue FormatParser::IOUtils::InvalidRead
+    nil
+  end
+
+  def read_and_unpack_packspec(io, **packspec)
+    sizes = {'a' => 1, 'N' => 4}
+    n = packspec.values.map { |e| sizes.fetch(e[0]) * e[1].to_i }.inject(&:+)
+    byte_str = safe_read(io, n)
+
+    unpacked_values = byte_str.unpack(packspec.values.join)
+    Hash[packspec.keys.zip(unpacked_values)]
+  end
+
+  def parse_id3_v2_header(io)
+    fields = {tag: :a3, version: :a2, flags: :a1, syncsafe_size: :N1}
+    header_data = read_and_unpack_packspec(io, **fields)
+    header_data[:size] = ID3Tag::SynchsafeInteger.decode(header_data.delete(:syncsafe_size))
+    header_data
+  end
+
+  # We swallow exceptions from ID3Tag primarily because it does not have
+  # a single wrapping error class we could capture. We also do not touch our original
+  # IO object when working with ID3Tag
+  def swallow_exceptions
+    yield
+  rescue => e
+    warn(e)
+    nil
+  end
+
+  extend self
+end

data/lib/parsers/zip_parser.rb

@@ -1,5 +1,4 @@
 class FormatParser::ZIPParser
-  UNICODE_REPLACEMENT_CHAR = [0xFFFD].pack('U')
   require_relative 'zip_parser/file_reader'
   require_relative 'zip_parser/office_formats'
 
@@ -40,7 +39,7 @@ class FormatParser::ZIPParser
 
   def decode_filename(filename, likely_unicode:)
     filename.force_encoding(Encoding::UTF_8) if likely_unicode
-    filename.encode(Encoding::UTF_8, undef: :replace, replace: UNICODE_REPLACEMENT_CHAR)
+    FormatParser.string_to_lossy_utf8(filename)
   end
 
   def decode_filename_of(zip_entry)

data/spec/parsers/mp3_parser_spec.rb

@@ -29,6 +29,24 @@ describe FormatParser::MP3Parser do
     expect(parsed.media_duration_seconds).to be_within(0.1).of(0.81)
   end
 
+  it 'does not attempt to read ID3V2 tags that are too large' do
+    more_bytes_than_permitted = 3 * 1024 * 1024
+    gunk = Random.new.bytes(more_bytes_than_permitted)
+
+    large_syncsfe_size = [ID3Tag::SynchsafeInteger.encode(more_bytes_than_permitted)].pack('N')
+    prepped = StringIO.new(
+      'ID3' + "\x43\x00".b + "\x00".b + large_syncsfe_size + gunk
+    )
+
+    expect(ID3Tag).not_to receive(:read)
+
+    prepped.seek(0)
+    result = FormatParser::MP3Parser::ID3Extraction.attempt_id3_v2_extraction(prepped)
+
+    expect(result).to be_nil
+    expect(prepped.pos).to eq(3145738)
+  end
+
   it 'parses the Cassy MP3' do
     fpath = fixtures_dir + '/MP3/Cassy.mp3'
     parsed = subject.call(File.open(fpath, 'rb'))
@@ -39,10 +57,21 @@ describe FormatParser::MP3Parser do
     expect(parsed.format).to eq(:mp3)
     expect(parsed.num_audio_channels).to eq(2)
     expect(parsed.audio_sample_rate_hz).to eq(44100)
-    expect(parsed.intrinsics).not_to be_nil
     expect(parsed.media_duration_seconds).to be_within(0.1).of(1102.46)
 
     expect(parsed.intrinsics).not_to be_nil
+
+    i = parsed.intrinsics
+    expect(i[:artist]).to eq('WeTransfer Studios/GIlles Peterson')
+    expect(i[:title]).to eq('Cassy')
+    expect(i[:album]).to eq('The Psychology of DJing')
+    expect(i[:comments]).to eq('0')
+    expect(i[:id3tags]).not_to be_nil
+
+    expect(parsed.intrinsics).not_to be_nil
+
+    # Make sure we are good with our JSON representation as well
+    JSON.pretty_generate(parsed)
   end
 
   it 'avoids returning a result when the parsed duration is infinite' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.12.4
+  version: 0.13.0
 platform: ruby
 authors:
 - Noah Berman
@@ -39,6 +39,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: id3tag
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -186,8 +200,7 @@ files:
 - lib/parsers/moov_parser.rb
 - lib/parsers/moov_parser/decoder.rb
 - lib/parsers/mp3_parser.rb
-- lib/parsers/mp3_parser/id3_v1.rb
-- lib/parsers/mp3_parser/id3_v2.rb
+- lib/parsers/mp3_parser/id3_extraction.rb
 - lib/parsers/pdf_parser.rb
 - lib/parsers/png_parser.rb
 - lib/parsers/psd_parser.rb

data/lib/parsers/mp3_parser/id3_v1.rb

@@ -1,48 +0,0 @@
-module FormatParser::MP3Parser::ID3V1
-  PACKSPEC = [
-    :tag, :a3,
-    :song_name, :a30,
-    :artist, :a30,
-    :album, :a30,
-    :year, :N1,
-    :comment, :a30,
-    :genre, :C,
-  ]
-  packspec_keys = PACKSPEC.select.with_index { |_, i| i.even? }
-  TAG_SIZE_BYTES = 128
-
-  class TagInformation < Struct.new(*packspec_keys)
-  end
-
-  def attempt_id3_v1_extraction(io)
-    return if io.size < TAG_SIZE_BYTES # Won't fit the ID3v1 regardless
-
-    io.seek(io.size - 128)
-    trailer_bytes = io.read(128)
-
-    return unless trailer_bytes && trailer_bytes.byteslice(0, 3) == 'TAG'
-
-    id3_v1 = parse_id3_v1(trailer_bytes)
-
-    # If all of the resulting strings are empty this ID3v1 tag is invalid and
-    # we should ignore it.
-    strings_from_id3v1 = id3_v1.values.select { |e| e.is_a?(String) && e != 'TAG' }
-    return if strings_from_id3v1.all?(&:empty?)
-
-    id3_v1
-  end
-
-  def parse_id3_v1(byte_str)
-    _keys, values = PACKSPEC.partition.with_index { |_, i| i.even? }
-    unpacked_values = byte_str.unpack(values.join)
-    unpacked_values.map! { |e| e.is_a?(String) ? trim_id3v1_string(e) : e }
-    TagInformation.new(unpacked_values)
-  end
-
-  # Remove trailing whitespace and trailing nullbytes
-  def trim_id3v1_string(str)
-    str.tr("\x00".b, '').strip
-  end
-
-  extend self
-end

data/lib/parsers/mp3_parser/id3_v2.rb

@@ -1,84 +0,0 @@
-module FormatParser::MP3Parser::ID3V2
-  def attempt_id3_v2_extraction(io)
-    io.seek(0) # Only support header ID3v2
-    header_bytes = io.read(10)
-    return unless header_bytes
-
-    header = parse_id3_v2_header(header_bytes)
-    return unless header[:tag] == 'ID3'
-    return unless header[:size] > 0
-
-    header_tag_payload = io.read(header[:size])
-    header_tag_payload = StringIO.new(header_tag_payload)
-
-    return unless header_tag_payload.size == header[:size]
-
-    frames = []
-    loop do
-      break if header_tag_payload.eof?
-      frame = parse_id3_v2_frame(header_tag_payload)
-      # Some files include padding, which is there so that when you edit ID3v2
-      # you do not have to overwrite the entire file - you can use this padding to
-      # add some more tags or to grow the existing ones. In practice if we hit
-      # something with a type of "0x00000000" we have entered the padding zone and
-      # there is no point in parsing further
-      if frame[:id] == "\x00\x00\x00\x00".b
-        break
-      else
-        frames << frame
-      end
-    end
-    frames
-  end
-
-  def parse_id3_v2_header(byte_str)
-    packspec = [
-      :tag, :a3,
-      :version, :a2,
-      :flags, :C1,
-      :size, :a4,
-    ]
-    keys, values = packspec.partition.with_index { |_, i| i.even? }
-    unpacked_values = byte_str.unpack(values.join)
-    header_data = Hash[keys.zip(unpacked_values)]
-
-    header_data[:version] = header_data[:version].unpack('C2')
-    header_data[:size] = decode_syncsafe_int(header_data[:size])
-
-    header_data
-  end
-
-  def parse_id3_v2_frame(io)
-    id, syncsafe_size, flags = io.read(10).unpack('a4a4a2')
-    size = decode_syncsafe_int(syncsafe_size)
-    content = io.read(size)
-    # It might so happen in sutations of terrible invalidity that we end up
-    # with less data than advertised by the syncsafe size. We will just truck on.
-    {id: id, size: size, flags: flags, content: content}
-  end
-
-  # ID3v2 uses "unsynchronized integers", which are unsigned integers smeared
-  # over multiple bytes in such a manner that the first bit is always 0 (unset).
-  # This is done so that ID3v2 incompatible decoders will not by accident see
-  # the 0xFF0xFF0xFF0xFF sequence anywhere that can be mistaken for the MPEG frame
-  # synchronisation header. Effectively it is a 7 bit big-endian unsigned integer
-  # encoding.
-  #
-  # 8 bit 255 (0xFF) encoded in this mannner takes 16 bits instead,
-  # and looks like this: `0b00000001 01111111`. Note how it avoids having
-  # the first bit of the second byte be 1.
-  # This method decodes an unsigned integer packed in this fashion
-  def decode_syncsafe_int(bytes)
-    size = 0
-    j = 0
-    i = bytes.bytesize - 1
-    while i >= 0
-      size += 128**i * (bytes.getbyte(j) & 0x7f)
-      j += 1
-      i -= 1
-    end
-    size
-  end
-
-  extend self
-end