format_parser 0.21.0 → 0.23.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ad2245de4a2119c7572c3962ad14abbf16395b2bec4064b218ee9f99d1e7c24b
-  data.tar.gz: b982bcc7f6626b66684db532317b0c0d35cd062aa89766ea1a230f93e7d996d6
+  metadata.gz: f76db376646606abe6a7ccaa6f0a14efdc997ecd7fe29aff0ab3d8172857649f
+  data.tar.gz: 07c142e7ce6aaa518285d425eb95961d18e053a6b15456d4cb569cdc70a79069
 SHA512:
-  metadata.gz: 2df2a3763e12e2bb0c70a8f5ec3319fcf6a3210a73461dc8abb5ec2af706028403eca48cf8c589bd40122dff0cfdabce383a79b1253237e219a5c89936ec0a5e
-  data.tar.gz: 0ca6084649313b2c7ad32204b4c8b745f13dbf6cf2c347cb7c80ca7976b964f00feabd9666c0372e9a6c10e4ed250500ed4e202cdb687372cbd369378b7a0faa
+  metadata.gz: 3e92625dbe822d3423a084174fd7bc9d4a296effe26182a91743b36995f8470edc4fa56a3a0a10ce095d4998da5dee9620b983c1f980e0418a3b4574244e848c
+  data.tar.gz: b89df24f7b6638b3cd42b284a1792475a094e947bd995e5ccdd0c554014075f5d167a7c04fbc08c681483b0c82aa2e59ae4e4af62f99d6b158fb4b3f1096d80b

data/.gitignore

@@ -58,3 +58,6 @@ Gemfile.lock
 
 # OSX Files
 .DS_Store
+
+# rspec examples
+spec/examples.txt

data/CHANGELOG.md

@@ -1,3 +1,24 @@
+## 0.23.1
+* Updated gem exifr to fix problems related to jpeg files from Olympos microscopes, which often have bad thumbnail data
+
+## 0.23.0
+* Add ActiveStorage analyzer which can analyze ActiveStorage blobs. Enable it by setting
+  `config.active_storage.analyzers.prepend FormatParser::ActiveStorage::BlobAnalyzer`
+* Ignore empty ID3 tags and do not allow them to overwrite others
+* Update the id3tag dependency so that we can fallback to UTF8 instead of raising an error when parsing
+  MP3 files
+
+## 0.22.1
+* Fix Zip parser to not raise error for invalid zip files, with an invalid central directory
+
+## 0.22.0
+* Adds option `stringify_keys: true` to #as_json methods (fix #151)
+
+## 0.21.1
+* MPEG: Ensure parsing does not inadvertently return an Integer instead of Result|nil
+* MPEG: Scan further into the MPEG file than previously (scan 32 1KB chunks)
+* MPEG: Ensure the parser does not raise an exception when there is no data to read for scanning beyound the initial header
+
 ## 0.21.0
 * Adds support for MPEG video files
 

data/CONTRIBUTING.md

@@ -234,4 +234,9 @@ This provision also applies to the test files you include with the changed code
 
 ## Changelog
 
-When creating a new release you must add an entry in the `CHANGELOG.md`.
+When creating a new release you must add an entry in the `CHANGELOG.md`.
+
+## Testing locally
+
+It's possible to run `exe/format_parser_inspect FILE_NAME` or `exe/format_parser_inspect FILE_URI`
+to test the new code without the necessity of installing the gem.

data/README.md

@@ -75,6 +75,17 @@ img_info = FormatParser.parse(File.open("myimage.jpg", "rb"))
 JSON.pretty_generate(img_info) #=> ...
 ```
 
+To convert the result to a Hash or a structure suitable for JSON serialization
+
+```ruby
+img_info = FormatParser.parse(File.open("myimage.jpg", "rb"))
+img_info.as_json
+
+# it's also possible to convert all keys to string
+img_info.as_json(stringify_keys: true)
+```
+
+
 ## Creating your own parsers
 
 See the [section on writing parsers in CONTRIBUTING.md](CONTRIBUTING.md#so-you-want-to-contribute-a-new-parser)
@@ -188,7 +199,7 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
 
 ## Copyright
 
-Copyright (c) 2019 WeTransfer. 
+Copyright (c) 2020 WeTransfer.
 
 `format_parser` is distributed under the conditions of the [Hippocratic License](https://firstdonoharm.dev/version/1/2/license.html)
   - See LICENSE.txt for further details.

data/format_parser.gemspec

@@ -31,15 +31,14 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'ks', '~> 0.0'
-  spec.add_dependency 'exifr', '~> 1', '>= 1.3.4'
-  spec.add_dependency 'id3tag', '~> 0.10', '>= 0.10.1'
+  spec.add_dependency 'exifr', '~> 1', '>= 1.3.7'
+  spec.add_dependency 'id3tag', '~> 0.13'
   spec.add_dependency 'faraday', '~> 0.13'
   spec.add_dependency 'measurometer', '~> 1'
 
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rake', '~> 12'
   spec.add_development_dependency 'simplecov', '~> 0.15'
-  spec.add_development_dependency 'pry', '~> 0.11'
   spec.add_development_dependency 'yard', '~> 0.9'
   spec.add_development_dependency 'wetransfer_style', '0.5.0'
   spec.add_development_dependency 'parallel_tests'

data/lib/active_storage/blob_analyzer.rb

@@ -0,0 +1,35 @@
+require_relative 'blob_io'
+
+# An analyzer class that can be hooked to ActiveStorage, in order to enable
+# FormatParser to do the blob analysis instead of ActiveStorage builtin-analyzers.
+# Invoked if properly integrated in Rails initializer.
+
+module FormatParser
+  module ActiveStorage
+    class BlobAnalyzer
+      # Format parser is able to handle a lot of format so by default it will accept all files
+      #
+      # @return [Boolean, true] always return true
+      def self.accept?(_blob)
+        true
+      end
+
+      def initialize(blob)
+        @blob = blob
+      end
+
+      # @return [Hash] file metadatas
+      def metadata
+        io = BlobIO.new(@blob)
+        parsed_file = FormatParser.parse(io)
+
+        if parsed_file
+          # We symbolize keys because of existing output hash format of ImageAnalyzer
+          parsed_file.as_json.symbolize_keys
+        else
+          logger.info "Skipping file analysis because FormatParser doesn't support the file"
+        end
+      end
+    end
+  end
+end

data/lib/active_storage/blob_io.rb

@@ -0,0 +1,51 @@
+# Acts as a proxy to turn ActiveStorage file into IO object
+
+module FormatParser
+  module ActiveStorage
+    class BlobIO
+      # @param blob[ActiveStorage::Blob] the file with linked service
+      # @return [BlobIO]
+      def initialize(blob)
+        @blob = blob
+        @service = blob.service
+        @pos = 0
+      end
+
+      # Emulates IO#read, but requires the number of bytes to read.
+      # Rely on `ActiveStorage::Service.download_chunk` of each hosting type (local, S3, Azure, etc)
+      #
+      # @param n_bytes[Integer] how many bytes to read
+      # @return [String] the read bytes
+      def read(n_bytes)
+        # HTTP ranges are exclusive.
+        http_range = (@pos..(@pos + n_bytes - 1))
+        body = @service.download_chunk(@blob.key, http_range)
+        @pos += body.bytesize
+        body.force_encoding(Encoding::ASCII_8BIT)
+      end
+
+      # Emulates IO#seek
+      #
+      # @param [Integer] offset size
+      # @return [Integer] always return 0, `seek` only mutates `pos` attribute
+      def seek(offset)
+        @pos = offset
+        0
+      end
+
+      # Emulates IO#size.
+      #
+      # @return [Integer] the size of the blob size from ActiveStorage
+      def size
+        @blob.byte_size
+      end
+
+      # Emulates IO#pos
+      #
+      # @return [Integer] the current offset (in bytes) of the io
+      def pos
+        @pos
+      end
+    end
+  end
+end

data/lib/attributes_json.rb

@@ -15,7 +15,12 @@ module FormatParser::AttributesJSON
 
   # Implements a sane default `as_json` for an object
   # that accessors defined
-  def as_json(root: false)
+  #
+  # @param root[Bool] if true, it surrounds the result in a hash with a key
+  #  `format_parser_file_info`
+  # @param stringify_keys[Bool] if true, it transforms all the hash keys to a string.
+  #   The default value is false for backward compatibility
+  def as_json(root: false, stringify_keys: false, **)
     h = {}
     h['nature'] = nature if respond_to?(:nature) # Needed for file info structs
     methods.grep(/\w\=$/).each_with_object(h) do |attr_writer_method_name, h|
@@ -27,6 +32,9 @@ module FormatParser::AttributesJSON
       sanitized_value = _sanitize_json_value(unwrapped_attribute_value)
       h[reader_method_name] = sanitized_value
     end
+
+    h = FormatParser::HashUtils.deep_transform_keys(h, &:to_s) if stringify_keys
+
     if root
       {'format_parser_file_info' => h}
     else

data/lib/format_parser.rb

@@ -5,6 +5,7 @@ require 'measurometer'
 # top-level methods of the library.
 module FormatParser
   require_relative 'format_parser/version'
+  require_relative 'hash_utils'
   require_relative 'attributes_json'
   require_relative 'image'
   require_relative 'audio'
@@ -17,6 +18,7 @@ module FormatParser
   require_relative 'remote_io'
   require_relative 'io_constraint'
   require_relative 'care'
+  require_relative 'active_storage/blob_analyzer'
 
   # Define Measurometer in the internal namespace as well
   # so that we stay compatible for the applications that use it

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.21.0'
+  VERSION = '0.23.1'
 end

data/lib/hash_utils.rb

@@ -0,0 +1,19 @@
+# based on https://github.com/rails/rails/blob/master/activesupport/lib/active_support/core_ext/hash/keys.rb#L116
+# I chose to copy this method instead of adding activesupport as a dependency
+# because we want to have the least number of dependencies
+module FormatParser
+  class HashUtils
+    def self.deep_transform_keys(object, &block)
+      case object
+      when Hash
+        object.each_with_object({}) do |(key, value), result|
+          result[yield(key)] = deep_transform_keys(value, &block)
+        end
+      when Array
+        object.map { |e| deep_transform_keys(e, &block) }
+      else
+        object
+      end
+    end
+  end
+end

data/lib/parsers/mp3_parser.rb

@@ -44,13 +44,9 @@ class FormatParser::MP3Parser
       tag = __getobj__
       MEMBERS.each_with_object({}) do |k, h|
         value = tag.public_send(k)
-        h[k] = value if value
+        h[k] = value if value && !value.empty?
       end
     end
-
-    def as_json(*)
-      to_h
-    end
   end
 
   def likely_match?(filename)
@@ -85,7 +81,7 @@ class FormatParser::MP3Parser
 
     first_frame = initial_frames.first
 
-    id3tags_hash = blend_id3_tags_into_hash(*tags)
+    id3tags_hash = with_id3tag_local_configs { blend_id3_tags_into_hash(*tags) }
 
     file_info = FormatParser::Audio.new(
       format: :mp3,
@@ -297,5 +293,14 @@ class FormatParser::MP3Parser
     attrs
   end
 
+  def with_id3tag_local_configs
+    ID3Tag.local_configuration do |c|
+      c.string_encode_options = { invalid: :replace, undef: :replace }
+      c.source_encoding_fallback = Encoding::UTF_8
+
+      yield
+    end
+  end
+
   FormatParser.register_parser new, natures: :audio, formats: :mp3, priority: 99
 end

data/lib/parsers/mpeg_parser.rb

@@ -26,9 +26,8 @@ class FormatParser::MPEGParser
 
   PACK_HEADER_START_CODE = [0x00, 0x00, 0x01, 0xBA].pack('C*')
   SEQUENCE_HEADER_START_CODE = [0xB3].pack('C*')
-  SEEK_FOR_SEQUENCE_HEADER_TIMES_LIMIT = 4
-  SEEK_FOR_SEQUENCE_HEADER_START_CODE_TIMES_LIMIT = 4
-  BYTES_TO_READ_PER_TIME = 1024
+  MAX_BLOCK_READS = 32
+  BYTES_TO_READ_PER_READ = 1024
 
   def self.likely_match?(filename)
     filename =~ /\.(mpg|mpeg)$/i
@@ -37,18 +36,19 @@ class FormatParser::MPEGParser
   def self.call(io)
     return unless matches_mpeg_header?(io)
 
-    # We are looping though the stream because there can be several sequence headers and some of them are not usefull.
-    # If we detect that the header is not usefull, then we look for the next one for SEEK_FOR_SEQUENCE_HEADER_TIMES_LIMIT
+    # We are looping though the stream because there can be several sequence headers and some of them are not useful.
+    # If we detect that the header is not useful, then we look for the next one for SEEK_FOR_SEQUENCE_HEADER_TIMES_LIMIT
     # If we reach the EOF, then the mpg is likely to be corrupted and we return nil
-    SEEK_FOR_SEQUENCE_HEADER_TIMES_LIMIT.times do
-      return if fetch_next_sequence_header_code_position(io).nil?
+    MAX_BLOCK_READS.times do
+      next unless pos = find_next_header_code_pos(io)
+      io.seek(pos + 1)
       horizontal_size, vertical_size = parse_image_size(io)
       ratio_code, rate_code = parse_rate_information(io)
-
       if valid_aspect_ratio_code?(ratio_code) && valid_frame_rate_code?(rate_code)
         return file_info(horizontal_size, vertical_size, ratio_code, rate_code)
       end
     end
+    nil # otherwise the return value of Integer#times will be returned
   rescue FormatParser::IOUtils::InvalidRead
     nil
   end
@@ -90,15 +90,11 @@ class FormatParser::MPEGParser
   # Returns the position of the next sequence package content in the stream
   # This method will read BYTES_TO_READ_PER_TIME in each loop for a maximum amount of SEEK_FOR_SEQUENCE_HEADER_START_CODE_TIMES_LIMIT times
   # If the package is not found, then it returns nil.
-  def self.fetch_next_sequence_header_code_position(io)
-    SEEK_FOR_SEQUENCE_HEADER_START_CODE_TIMES_LIMIT.times do
-      bytes_stream_read = io.read(BYTES_TO_READ_PER_TIME)
-      header_relative_index = bytes_stream_read.index(SEQUENCE_HEADER_START_CODE)
-      next if header_relative_index.nil?
-      new_io_pos = io.pos - BYTES_TO_READ_PER_TIME + header_relative_index + 1
-      io.seek(new_io_pos)
-      return new_io_pos
-    end
+  def self.find_next_header_code_pos(io)
+    pos_before_read = io.pos
+    bin_str = io.read(BYTES_TO_READ_PER_READ) # bin_str might be nil if we are at EOF
+    header_relative_index = bin_str && bin_str.index(SEQUENCE_HEADER_START_CODE)
+    return pos_before_read + header_relative_index if header_relative_index
   end
 
   # If the first 4 bytes of the stream are equal to 00 00 01 BA, the pack start code for the Pack Header, then it's an MPEG file.

data/lib/parsers/zip_parser/file_reader.rb

@@ -18,6 +18,7 @@ class FormatParser::ZIPParser::FileReader
       'Could not find the EOCD signature in the buffer - maybe a malformed ZIP file'
     end
   end
+  InvalidCentralDirectory = Class.new(Error)
 
   C_UINT32LE = 'V'
   C_UINT16LE = 'v'
@@ -175,6 +176,8 @@ class FormatParser::ZIPParser::FileReader
     # BUT! in format_parser we avoid unbounded reads, as a matter of fact they are forbidden.
     # So we will again limit ouselves to cdir_size, and we will take cushion of 1 KB.
     central_directory_str = io.read(cdir_size + 1024)
+    raise InvalidCentralDirectory if central_directory_str.nil?
+
     central_directory_io = StringIO.new(central_directory_str)
     log do
       format(

data/spec/active_storage/blob_io_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe FormatParser::ActiveStorage::BlobIO do
+  let(:blob_service) { double }
+  let(:blob) { double(key: 'blob_key', service: blob_service, byte_size: 43000) }
+  let(:io) { described_class.new(blob) }
+  let(:fixture_path) { fixtures_dir + '/test.png' }
+
+  it_behaves_like 'an IO object compatible with IOConstraint'
+
+  describe '#read' do
+    it 'reads io using download_chunk from ActiveStorage#Service' do
+      allow(blob_service).to receive(:download_chunk) { 'a' }
+
+      expect(io.read(1)).to eq('a')
+    end
+
+    it 'updates #pos on read' do
+      allow(blob_service).to receive(:download_chunk) { 'a' }
+
+      expect { io.read(1) }.to change { io.pos }.from(0).to(1)
+    end
+  end
+
+  describe '#seek' do
+    it 'updates @pos' do
+      expect { io.seek(10) }.to change { io.pos }.from(0).to(10)
+    end
+  end
+
+  describe '#size' do
+    it 'returns the size of the blob byte_size' do
+      expect(io.size).to eq(blob.byte_size)
+    end
+  end
+end

data/spec/active_storage/rails_app_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+
+def skip_reason
+  if RUBY_ENGINE == 'jruby'
+    'Skipping because JRuby have randon failing issue'
+  elsif RUBY_VERSION.to_f < 2.5
+    'Skipping because Rails testing script use Rails 6, who does not support Ruby bellow 2.5'
+  else
+    false
+  end
+end
+
+describe 'Rails app with ActiveStorage and format-parser', skip: skip_reason do
+  describe 'local hosting with ActiveStorage disk adapter' do
+    it 'parse local file with format_parser' do
+      clean_env do
+        cmd = 'ruby spec/integration/active_storage/rails_app.rb'
+        cmd_status = ruby_script_runner(cmd)
+        expect(cmd_status[:stdout].last).to match(/1 runs, 3 assertions, 0 failures, 0 errors, 0 skips/)
+        expect(cmd_status[:exitstatus]).to eq(0)
+      end
+    end
+  end
+
+  def ruby_script_runner(cmd)
+    require 'open3'
+    cmd_status = { stdout: [], exitstatus: nil }
+    Open3.popen2(cmd) do |_stdin, stdout, wait_thr|
+      frame_stdout do
+        while line = stdout.gets
+          puts "|  #{line}"
+          cmd_status[:stdout] << line
+        end
+      end
+      cmd_status[:exitstatus] = wait_thr.value.exitstatus
+    end
+    cmd_status
+  end
+
+  def frame_stdout
+    puts
+    puts '-' * 50
+    yield
+    puts '-' * 50
+  end
+
+  def clean_env
+    if Bundler.respond_to?(:with_unbundled_env)
+      Bundler.with_unbundled_env do
+        yield
+      end
+    else
+      Bundler.with_clean_env do
+        yield
+      end
+    end
+  end
+end

data/spec/attributes_json_spec.rb

@@ -140,4 +140,30 @@ describe FormatParser::AttributesJSON do
       JSON.pretty_generate(object_with_attributes_module)
     }.to raise_error(/structure too deep/)
   end
+
+  it 'converts all hash keys to string when stringify_keys: true' do
+    fixture_path = fixtures_dir + '/ZIP/arch_few_entries.zip'
+    fi_io = File.open(fixture_path, 'rb')
+
+    result = FormatParser::ZIPParser.new.call(fi_io).as_json(stringify_keys: true)
+
+    result['entries'].each do |entry|
+      entry.each do |key, _value|
+        expect(key).to be_a(String)
+      end
+    end
+  end
+
+  it 'does not convert hash keys to string when stringify_keys: false' do
+    fixture_path = fixtures_dir + '/ZIP/arch_few_entries.zip'
+    fi_io = File.open(fixture_path, 'rb')
+
+    result = FormatParser::ZIPParser.new.call(fi_io).as_json
+
+    result['entries'].each do |entry|
+      entry.each do |key, _value|
+        expect(key).to be_a(Symbol)
+      end
+    end
+  end
 end

data/spec/hash_utils_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+
+describe FormatParser::HashUtils do
+  describe '.deep_transform_keys' do
+    it 'transforms all the keys in a hash' do
+      hash = { aa: 1, 'bb' => 2 }
+      result = described_class.deep_transform_keys(hash, &:to_s)
+
+      expect(result).to eq('aa' => 1, 'bb' => 2)
+    end
+
+    it 'transforms all the keys in a array of hashes' do
+      array = [{ aa: 1, bb: 2 }, { cc: 3, dd: [{c: 2, d: 3}] }]
+      result = described_class.deep_transform_keys(array, &:to_s)
+
+      expect(result).to eq(
+        [{'aa' => 1, 'bb' => 2}, {'cc' => 3, 'dd' => [{'c' => 2, 'd' => 3}]}]
+      )
+    end
+
+    it 'transforms all the keys in a hash recursively' do
+      hash = { aa: 1, bb: { cc: 22, dd: 3 } }
+      result = described_class.deep_transform_keys(hash, &:to_s)
+
+      expect(result).to eq('aa' => 1, 'bb' => { 'cc' => 22, 'dd' => 3})
+    end
+
+    it 'does nothing for an non array/hash object' do
+      object = Object.new
+      result = described_class.deep_transform_keys(object, &:to_s)
+
+      expect(result).to eq(object)
+    end
+
+    it 'returns the last value if different keys are transformed into the same one' do
+      hash = { aa: 0, 'bb' => 2, bb: 1 }
+      result = described_class.deep_transform_keys(hash, &:to_s)
+
+      expect(result).to eq('aa' => 0, 'bb' => 1)
+    end
+  end
+end

data/spec/integration/active_storage/rails_app.rb

@@ -0,0 +1,72 @@
+require 'bundler/inline'
+
+gemfile(true) do
+  source 'https://rubygems.org'
+
+  git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+  gem 'rails', '6.0.3'
+  gem 'sqlite3'
+  gem 'format_parser', path: './'
+end
+
+require 'active_record/railtie'
+require 'active_storage/engine'
+require 'tmpdir'
+
+class TestApp < Rails::Application
+  config.root = __dir__
+  config.hosts << 'example.org'
+  config.eager_load = false
+  config.session_store :cookie_store, key: 'cookie_store_key'
+  secrets.secret_key_base = 'secret_key_base'
+
+  config.logger = Logger.new('/dev/null')
+
+  config.active_storage.service = :local
+  config.active_storage.service_configurations = {
+    local: {
+      root: Dir.tmpdir,
+      service: 'Disk'
+    }
+  }
+
+  config.active_storage.analyzers.prepend FormatParser::ActiveStorage::BlobAnalyzer
+end
+
+ENV['DATABASE_URL'] = 'sqlite3::memory:'
+
+Rails.application.initialize!
+
+require ActiveStorage::Engine.root.join('db/migrate/20170806125915_create_active_storage_tables.rb').to_s
+
+ActiveRecord::Schema.define do
+  CreateActiveStorageTables.new.change
+
+  create_table :users, force: true
+end
+
+class User < ActiveRecord::Base
+  has_one_attached :profile_picture
+end
+
+require 'minitest/autorun'
+require 'open-uri'
+
+describe User do
+  describe "profile_picture's metadatas" do
+    it 'parse metadatas with format_parser' do
+      user = User.create
+      user.profile_picture.attach(
+        filename: 'cat.png',
+        io: URI.open('https://freesvg.org/img/1416155153.png')
+      )
+
+      user.profile_picture.analyze
+
+      _(user.profile_picture.metadata[:width_px]).must_equal 500
+      _(user.profile_picture.metadata[:height_px]).must_equal 296
+      _(user.profile_picture.metadata[:color_mode]).must_equal 'rgba'
+    end
+  end
+end

data/spec/parsers/mp3_parser_spec.rb

@@ -37,6 +37,14 @@ describe FormatParser::MP3Parser do
         expect(parsed.album).to be_nil
       end
     end
+
+    context 'when has an empty tag' do
+      let(:fpath) { fixtures_dir + '/MP3/id3v2_with_empty_tag.mp3' }
+
+      it 'ignores the empty tags' do
+        expect(parsed.intrinsics[:genre]).to eq('Rock')
+      end
+    end
   end
 
   it 'decodes and estimates duration for a CBR MP3' do
@@ -71,6 +79,17 @@ describe FormatParser::MP3Parser do
     expect(prepped.pos).to eq(3145738)
   end
 
+  it 'does not raise error when a tag frame has unsupported encoding' do
+    fpath = fixtures_dir + '/MP3/id3v2_frame_with_invalid_encoding.mp3'
+
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed.nature). to eq(:audio)
+    expect(parsed.album).to eq('wetransfer')
+    expect(parsed.artist).to eq('wetransfer')
+    expect(parsed.title).to eq('test')
+  end
+
   it 'parses the Cassy MP3' do
     fpath = fixtures_dir + '/MP3/Cassy.mp3'
     parsed = subject.call(File.open(fpath, 'rb'))
@@ -110,4 +129,32 @@ describe FormatParser::MP3Parser do
       subject.call(StringIO.new(''))
     }.to raise_error(FormatParser::IOUtils::InvalidRead)
   end
+
+  describe '#as_json' do
+    it 'converts all hash keys to string when stringify_keys: true' do
+      fpath = fixtures_dir + '/MP3/Cassy.mp3'
+      result = subject.call(File.open(fpath, 'rb')).as_json(stringify_keys: true)
+
+      expect(
+        result['intrinsics'].keys.map(&:class).uniq
+      ).to eq([String])
+
+      expect(
+        result['intrinsics']['id3tags'].map(&:class).uniq
+      ).to eq([ID3Tag::Tag])
+    end
+
+    it 'does not convert the hash keys to string when stringify_keys: false' do
+      fpath = fixtures_dir + '/MP3/Cassy.mp3'
+      result = subject.call(File.open(fpath, 'rb')).as_json
+
+      expect(
+        result['intrinsics'].keys.map(&:class).uniq
+      ).to eq([Symbol])
+
+      expect(
+        result['intrinsics'][:id3tags].map(&:class).uniq
+      ).to eq([ID3Tag::Tag])
+    end
+  end
 end

data/spec/parsers/mpeg_parser_spec.rb

@@ -12,6 +12,27 @@ describe FormatParser::MPEGParser do
     expect(parse_result.intrinsics[:frame_rate]).to eq('30')
   end
 
+  it 'returns a nil if it is necessary to iterate over a very large number of bytes and the requisite sequences are not detected' do
+    bytes_buffer = StringIO.new
+    bytes_buffer.write([0x00, 0x00, 0x01, 0xBA].pack('C*')) # MPEG header
+    zero_bytes = [0x00].pack('C') * (1024 * 1024 * 5)
+    bytes_buffer.write(zero_bytes)
+
+    bytes_buffer.rewind
+
+    parse_result = described_class.call(bytes_buffer)
+    expect(parse_result).to be_nil
+  end
+
+  it 'returns a nil if the IO only contains the MPEG header bytes at the start and nothing else' do
+    bytes_buffer = StringIO.new
+    bytes_buffer.write([0x00, 0x00, 0x01, 0xBA].pack('C*')) # MPEG header
+    bytes_buffer.rewind
+
+    parse_result = described_class.call(bytes_buffer)
+    expect(parse_result).to be_nil
+  end
+
   it 'parses a file with mpeg extension' do
     parse_result = described_class.call(File.open(__dir__ + '/../fixtures/MPG/video2.mpeg', 'rb'))
 

data/spec/parsers/zip_parser_spec.rb

@@ -103,4 +103,11 @@ describe FormatParser::ZIPParser do
     expect(first_entry.filename).to eq('Li��nia Extreme//')
     expect(first_entry.type).to eq(:directory)
   end
+
+  it 'is able to handle files with invalid central directory position' do
+    invalid_zip_path = fixtures_dir + '/ZIP/invalid_central_directory.zip'
+
+    expect { subject.call(File.open(invalid_zip_path, 'rb')) }
+      .to_not raise_error
+  end
 end

data/spec/spec_helper.rb

@@ -8,7 +8,6 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
 
 require 'rspec'
 require 'format_parser'
-require 'pry'
 
 module SpecHelpers
   def fixtures_dir
@@ -19,6 +18,8 @@ end
 RSpec.configure do |c|
   c.include SpecHelpers
   c.extend SpecHelpers # makes fixtures_dir available for example groups too
+  # https://relishapp.com/rspec/rspec-core/docs/command-line/only-failures
+  c.example_status_persistence_file_path = 'spec/examples.txt'
 end
 
 RSpec.shared_examples 'an IO object compatible with IOConstraint' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.21.0
+  version: 0.23.1
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-27 00:00:00.000000000 Z
+date: 2020-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -34,7 +34,7 @@ dependencies:
         version: '1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.4
+        version: 1.3.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -44,27 +44,21 @@ dependencies:
         version: '1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.4
+        version: 1.3.7
 - !ruby/object:Gem::Dependency
   name: id3tag
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.10.1
+        version: '0.13'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.10.1
+        version: '0.13'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -135,20 +129,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.15'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.11'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.11'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -216,6 +196,8 @@ files:
 - Rakefile
 - exe/format_parser_inspect
 - format_parser.gemspec
+- lib/active_storage/blob_analyzer.rb
+- lib/active_storage/blob_io.rb
 - lib/archive.rb
 - lib/attributes_json.rb
 - lib/audio.rb
@@ -223,6 +205,7 @@ files:
 - lib/document.rb
 - lib/format_parser.rb
 - lib/format_parser/version.rb
+- lib/hash_utils.rb
 - lib/image.rb
 - lib/io_constraint.rb
 - lib/io_utils.rb
@@ -254,12 +237,16 @@ files:
 - lib/read_limits_config.rb
 - lib/remote_io.rb
 - lib/video.rb
+- spec/active_storage/blob_io_spec.rb
+- spec/active_storage/rails_app_spec.rb
 - spec/attributes_json_spec.rb
 - spec/care_spec.rb
 - spec/esoteric_formats_spec.rb
 - spec/file_information_spec.rb
 - spec/format_parser_inspect_spec.rb
 - spec/format_parser_spec.rb
+- spec/hash_utils_spec.rb
+- spec/integration/active_storage/rails_app.rb
 - spec/io_utils_spec.rb
 - spec/parsers/aiff_parser_spec.rb
 - spec/parsers/bmp_parser_spec.rb