format_parser 0.22.1 → 0.24.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 05470fef0edec68c427b9680cf94a88a8f3286e2279b2daf13e87679324eb061
-  data.tar.gz: f65ed1d390e4e70bd34a3b01b5d54d39e4076d18ebf15fc20196c2746c86122f
+  metadata.gz: b680c35747790df90503edc5cb7ddda0d8929ab47c945fe6c13f5fa4319ba747
+  data.tar.gz: e8cf15c34f0422a99f3e09b30ab7ca50d54c17f2933762414323f34f06100260
 SHA512:
-  metadata.gz: e6e9e2d1fbfe6e813d1e0d9d2c102ce3f031fd37395cc75807ca7b527b1a2fc4a9a3c41baaf9e2cdaa601063cded92ef446ce913f2210f3d82e063126873d54d
-  data.tar.gz: 86e166b2ac754fe5d7e42e471bcaa70cd54ae0aa22da3e0f85a6223213c2bd5a284f2b53416d03656dc291898837636678dbfba2e32b179a562ea7c8b05ef0bf
+  metadata.gz: 658e3d789e95c20b7a30e4a4c0cdb37fc186b0b696d8bfa1fd7b7b500bb375dfc0dc6873f908e20c45e871ac3831231adad3589457073bdf13697c79677b450f
+  data.tar.gz: 669af5c03327b8e048cee8401ac340b77b5bb990d08b4786c402451a092fdbea1d4a106a6404a6ac711baed2171eced775b37e5e5979e130fea2e833380e7598

data/CHANGELOG.md

@@ -1,3 +1,23 @@
+## 0.24.2
+* Update gem id3tag to 0.14.0 to fix MP3 issues
+
+## 0.24.1
+* Fix MP3 frames reading to jump correctly to the next bytes
+
+## 0.24.0
+* The TIFF parser will now return :arw as format for Sony ARW files insted of :tif so that the caller can decide whether it
+  wants to deal with RAW processing or not
+
+## 0.23.1
+* Updated gem exifr to fix problems related to jpeg files from Olympos microscopes, which often have bad thumbnail data
+
+## 0.23.0
+* Add ActiveStorage analyzer which can analyze ActiveStorage blobs. Enable it by setting
+  `config.active_storage.analyzers.prepend FormatParser::ActiveStorage::BlobAnalyzer`
+* Ignore empty ID3 tags and do not allow them to overwrite others
+* Update the id3tag dependency so that we can fallback to UTF8 instead of raising an error when parsing
+  MP3 files
+
 ## 0.22.1
 * Fix Zip parser to not raise error for invalid zip files, with an invalid central directory
 

data/format_parser.gemspec

@@ -31,8 +31,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'ks', '~> 0.0'
-  spec.add_dependency 'exifr', '~> 1', '>= 1.3.4'
-  spec.add_dependency 'id3tag', '~> 0.10', '>= 0.10.1'
+  spec.add_dependency 'exifr', '~> 1', '>= 1.3.7'
+  spec.add_dependency 'id3tag', '~> 0.14'
   spec.add_dependency 'faraday', '~> 0.13'
   spec.add_dependency 'measurometer', '~> 1'
 

data/lib/active_storage/blob_analyzer.rb

@@ -0,0 +1,35 @@
+require_relative 'blob_io'
+
+# An analyzer class that can be hooked to ActiveStorage, in order to enable
+# FormatParser to do the blob analysis instead of ActiveStorage builtin-analyzers.
+# Invoked if properly integrated in Rails initializer.
+
+module FormatParser
+  module ActiveStorage
+    class BlobAnalyzer
+      # Format parser is able to handle a lot of format so by default it will accept all files
+      #
+      # @return [Boolean, true] always return true
+      def self.accept?(_blob)
+        true
+      end
+
+      def initialize(blob)
+        @blob = blob
+      end
+
+      # @return [Hash] file metadatas
+      def metadata
+        io = BlobIO.new(@blob)
+        parsed_file = FormatParser.parse(io)
+
+        if parsed_file
+          # We symbolize keys because of existing output hash format of ImageAnalyzer
+          parsed_file.as_json.symbolize_keys
+        else
+          logger.info "Skipping file analysis because FormatParser doesn't support the file"
+        end
+      end
+    end
+  end
+end

data/lib/active_storage/blob_io.rb

@@ -0,0 +1,51 @@
+# Acts as a proxy to turn ActiveStorage file into IO object
+
+module FormatParser
+  module ActiveStorage
+    class BlobIO
+      # @param blob[ActiveStorage::Blob] the file with linked service
+      # @return [BlobIO]
+      def initialize(blob)
+        @blob = blob
+        @service = blob.service
+        @pos = 0
+      end
+
+      # Emulates IO#read, but requires the number of bytes to read.
+      # Rely on `ActiveStorage::Service.download_chunk` of each hosting type (local, S3, Azure, etc)
+      #
+      # @param n_bytes[Integer] how many bytes to read
+      # @return [String] the read bytes
+      def read(n_bytes)
+        # HTTP ranges are exclusive.
+        http_range = (@pos..(@pos + n_bytes - 1))
+        body = @service.download_chunk(@blob.key, http_range)
+        @pos += body.bytesize
+        body.force_encoding(Encoding::ASCII_8BIT)
+      end
+
+      # Emulates IO#seek
+      #
+      # @param [Integer] offset size
+      # @return [Integer] always return 0, `seek` only mutates `pos` attribute
+      def seek(offset)
+        @pos = offset
+        0
+      end
+
+      # Emulates IO#size.
+      #
+      # @return [Integer] the size of the blob size from ActiveStorage
+      def size
+        @blob.byte_size
+      end
+
+      # Emulates IO#pos
+      #
+      # @return [Integer] the current offset (in bytes) of the io
+      def pos
+        @pos
+      end
+    end
+  end
+end

data/lib/format_parser.rb

@@ -18,6 +18,7 @@ module FormatParser
   require_relative 'remote_io'
   require_relative 'io_constraint'
   require_relative 'care'
+  require_relative 'active_storage/blob_analyzer'
 
   # Define Measurometer in the internal namespace as well
   # so that we stay compatible for the applications that use it

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.22.1'
+  VERSION = '0.24.2'
 end

data/lib/parsers/mp3_parser.rb

@@ -20,13 +20,14 @@ class FormatParser::MP3Parser
 
   # We limit the number of MPEG frames we scan
   # to obtain our duration estimation
-  MAX_FRAMES_TO_SCAN = 128
+  MAX_FRAMES_TO_SCAN = 500
 
   # Default frame size for mp3
   SAMPLES_PER_FRAME = 1152
 
   # For some edge cases
   ZIP_LOCAL_ENTRY_SIGNATURE = "PK\x03\x04\x14\x00".b
+  PNG_HEADER_BYTES = [137, 80, 78, 71, 13, 10, 26, 10].pack('C*')
 
   # Wraps the Tag object returned by ID3Tag in such
   # a way that a usable JSON representation gets
@@ -44,7 +45,7 @@ class FormatParser::MP3Parser
       tag = __getobj__
       MEMBERS.each_with_object({}) do |k, h|
         value = tag.public_send(k)
-        h[k] = value if value
+        h[k] = value if value && !value.empty?
       end
     end
   end
@@ -60,8 +61,12 @@ class FormatParser::MP3Parser
     # To avoid having that happen, we check for the PKZIP signature -
     # local entry header signature - at the very start of the file.
     # If the file is too small safe_read will fail too and the parser
-    # will terminate here.
-    return if safe_read(io, 6) == ZIP_LOCAL_ENTRY_SIGNATURE
+    # will terminate here. Same with PNGs. In the future
+    # we should implement "confidence" for MP3 as of all our formats
+    # it is by far the most lax.
+    header = safe_read(io, 8)
+    return if header.start_with?(ZIP_LOCAL_ENTRY_SIGNATURE)
+    return if header.start_with?(PNG_HEADER_BYTES)
 
     # Read all the ID3 tags (or at least attempt to)
     io.seek(0)
@@ -81,7 +86,7 @@ class FormatParser::MP3Parser
 
     first_frame = initial_frames.first
 
-    id3tags_hash = blend_id3_tags_into_hash(*tags)
+    id3tags_hash = with_id3tag_local_configs { blend_id3_tags_into_hash(*tags) }
 
     file_info = FormatParser::Audio.new(
       format: :mp3,
@@ -131,27 +136,28 @@ class FormatParser::MP3Parser
   # if you have a minute. https://pypi.python.org/pypi/tinytag
   def parse_mpeg_frames(io)
     mpeg_frames = []
+    bytes_to_read = 4
 
     MAX_FRAMES_TO_SCAN.times do |frame_i|
       # Read through until we can latch onto the 11 sync bits. Read in 4-byte
       # increments to save on read() calls
-      data = io.read(4)
+      data = io.read(bytes_to_read)
 
       # If we are at EOF - stop iterating
-      break unless data && data.bytesize == 4
+      break unless data && data.bytesize == bytes_to_read
 
       # Look for the sync pattern. It can be either the last byte being 0xFF,
       # or any of the 2 bytes in sequence being 0xFF and > 0xF0.
       four_bytes = data.unpack('C4')
       seek_jmp = sync_bytes_offset_in_4_byte_seq(four_bytes)
       if seek_jmp > 0
-        io.seek(io.pos + seek_jmp)
+        io.seek(io.pos - bytes_to_read + seek_jmp)
         next
       end
 
       # Once we are past that stage we have latched onto a sync frame header
       sync, conf, bitrate_freq, rest = four_bytes
-      frame_detail = parse_mpeg_frame_header(io.pos - 4, sync, conf, bitrate_freq, rest)
+      frame_detail = parse_mpeg_frame_header(io.pos - bytes_to_read, sync, conf, bitrate_freq, rest)
       mpeg_frames << frame_detail
 
       # There might be a xing header in the first frame that contains
@@ -166,7 +172,7 @@ class FormatParser::MP3Parser
         end
       end
       if frame_detail.frame_length > 1 # jump over current frame body
-        io.seek(io.pos + frame_detail.frame_length - 4)
+        io.seek(io.pos + frame_detail.frame_length - bytes_to_read)
       end
     end
     [nil, mpeg_frames]
@@ -293,5 +299,14 @@ class FormatParser::MP3Parser
     attrs
   end
 
+  def with_id3tag_local_configs
+    ID3Tag.local_configuration do |c|
+      c.string_encode_options = { invalid: :replace, undef: :replace }
+      c.source_encoding_fallback = Encoding::UTF_8
+
+      yield
+    end
+  end
+
   FormatParser.register_parser new, natures: :audio, formats: :mp3, priority: 99
 end

data/lib/parsers/tiff_parser.rb

@@ -26,7 +26,7 @@ class FormatParser::TIFFParser
     h = exif_data.height || exif_data.pixel_y_dimension
 
     FormatParser::Image.new(
-      format: :tif,
+      format: arw?(exif_data) ? :arw : :tif, # Specify format as arw for Sony ARW format images, else tif
       width_px: w,
       height_px: h,
       display_width_px: exif_data.rotated? ? h : w,
@@ -43,5 +43,11 @@ class FormatParser::TIFFParser
     safe_read(io, 2) == 'CR'
   end
 
+  # Similar to how exiftool determines the image type as ARW, we are implementing a check here
+  # https://github.com/exiftool/exiftool/blob/e969456372fbaf4b980fea8bb094d71033ac8bf7/lib/Image/ExifTool/Exif.pm#L929
+  def arw?(exif_data)
+    exif_data.compression == 6 && exif_data.new_subfile_type == 1 && exif_data.make == 'SONY'
+  end
+
   FormatParser.register_parser new, natures: :image, formats: :tif
 end

data/spec/active_storage/blob_io_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe FormatParser::ActiveStorage::BlobIO do
+  let(:blob_service) { double }
+  let(:blob) { double(key: 'blob_key', service: blob_service, byte_size: 43000) }
+  let(:io) { described_class.new(blob) }
+  let(:fixture_path) { fixtures_dir + '/test.png' }
+
+  it_behaves_like 'an IO object compatible with IOConstraint'
+
+  describe '#read' do
+    it 'reads io using download_chunk from ActiveStorage#Service' do
+      allow(blob_service).to receive(:download_chunk) { 'a' }
+
+      expect(io.read(1)).to eq('a')
+    end
+
+    it 'updates #pos on read' do
+      allow(blob_service).to receive(:download_chunk) { 'a' }
+
+      expect { io.read(1) }.to change { io.pos }.from(0).to(1)
+    end
+  end
+
+  describe '#seek' do
+    it 'updates @pos' do
+      expect { io.seek(10) }.to change { io.pos }.from(0).to(10)
+    end
+  end
+
+  describe '#size' do
+    it 'returns the size of the blob byte_size' do
+      expect(io.size).to eq(blob.byte_size)
+    end
+  end
+end

data/spec/active_storage/rails_app_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+
+def skip_reason
+  if RUBY_ENGINE == 'jruby'
+    'Skipping because JRuby have randon failing issue'
+  elsif RUBY_VERSION.to_f < 2.5
+    'Skipping because Rails testing script use Rails 6, who does not support Ruby bellow 2.5'
+  else
+    false
+  end
+end
+
+describe 'Rails app with ActiveStorage and format-parser', skip: skip_reason do
+  describe 'local hosting with ActiveStorage disk adapter' do
+    it 'parse local file with format_parser' do
+      clean_env do
+        cmd = 'ruby spec/integration/active_storage/rails_app.rb'
+        cmd_status = ruby_script_runner(cmd)
+        expect(cmd_status[:stdout].last).to match(/1 runs, 3 assertions, 0 failures, 0 errors, 0 skips/)
+        expect(cmd_status[:exitstatus]).to eq(0)
+      end
+    end
+  end
+
+  def ruby_script_runner(cmd)
+    require 'open3'
+    cmd_status = { stdout: [], exitstatus: nil }
+    Open3.popen2(cmd) do |_stdin, stdout, wait_thr|
+      frame_stdout do
+        while line = stdout.gets
+          puts "|  #{line}"
+          cmd_status[:stdout] << line
+        end
+      end
+      cmd_status[:exitstatus] = wait_thr.value.exitstatus
+    end
+    cmd_status
+  end
+
+  def frame_stdout
+    puts
+    puts '-' * 50
+    yield
+    puts '-' * 50
+  end
+
+  def clean_env
+    if Bundler.respond_to?(:with_unbundled_env)
+      Bundler.with_unbundled_env do
+        yield
+      end
+    else
+      Bundler.with_clean_env do
+        yield
+      end
+    end
+  end
+end

data/spec/integration/active_storage/rails_app.rb

@@ -0,0 +1,72 @@
+require 'bundler/inline'
+
+gemfile(true) do
+  source 'https://rubygems.org'
+
+  git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+  gem 'rails', '6.0.3'
+  gem 'sqlite3'
+  gem 'format_parser', path: './'
+end
+
+require 'active_record/railtie'
+require 'active_storage/engine'
+require 'tmpdir'
+
+class TestApp < Rails::Application
+  config.root = __dir__
+  config.hosts << 'example.org'
+  config.eager_load = false
+  config.session_store :cookie_store, key: 'cookie_store_key'
+  secrets.secret_key_base = 'secret_key_base'
+
+  config.logger = Logger.new('/dev/null')
+
+  config.active_storage.service = :local
+  config.active_storage.service_configurations = {
+    local: {
+      root: Dir.tmpdir,
+      service: 'Disk'
+    }
+  }
+
+  config.active_storage.analyzers.prepend FormatParser::ActiveStorage::BlobAnalyzer
+end
+
+ENV['DATABASE_URL'] = 'sqlite3::memory:'
+
+Rails.application.initialize!
+
+require ActiveStorage::Engine.root.join('db/migrate/20170806125915_create_active_storage_tables.rb').to_s
+
+ActiveRecord::Schema.define do
+  CreateActiveStorageTables.new.change
+
+  create_table :users, force: true
+end
+
+class User < ActiveRecord::Base
+  has_one_attached :profile_picture
+end
+
+require 'minitest/autorun'
+require 'open-uri'
+
+describe User do
+  describe "profile_picture's metadatas" do
+    it 'parse metadatas with format_parser' do
+      user = User.create
+      user.profile_picture.attach(
+        filename: 'cat.png',
+        io: URI.open('https://freesvg.org/img/1416155153.png')
+      )
+
+      user.profile_picture.analyze
+
+      _(user.profile_picture.metadata[:width_px]).must_equal 500
+      _(user.profile_picture.metadata[:height_px]).must_equal 296
+      _(user.profile_picture.metadata[:color_mode]).must_equal 'rgba'
+    end
+  end
+end

data/spec/parsers/mp3_parser_spec.rb

@@ -15,6 +15,12 @@ describe FormatParser::MP3Parser do
     expect(parsed.media_duration_seconds).to be_within(0.1).of(0.836)
   end
 
+  it 'does not misdetect a PNG' do
+    fpath = fixtures_dir + '/PNG/anim.png'
+    parsed = subject.call(File.open(fpath, 'rb'))
+    expect(parsed).to be_nil
+  end
+
   describe 'title/artist/album attributes' do
     let(:parsed) { subject.call(File.open(fpath, 'rb')) }
 
@@ -37,6 +43,14 @@ describe FormatParser::MP3Parser do
         expect(parsed.album).to be_nil
       end
     end
+
+    context 'when has an empty tag' do
+      let(:fpath) { fixtures_dir + '/MP3/id3v2_with_empty_tag.mp3' }
+
+      it 'ignores the empty tags' do
+        expect(parsed.intrinsics[:genre]).to eq('Rock')
+      end
+    end
   end
 
   it 'decodes and estimates duration for a CBR MP3' do
@@ -71,6 +85,25 @@ describe FormatParser::MP3Parser do
     expect(prepped.pos).to eq(3145738)
   end
 
+  it 'does not raise error when a tag frame has unsupported encoding' do
+    fpath = fixtures_dir + '/MP3/id3v2_frame_with_invalid_encoding.mp3'
+
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed.nature). to eq(:audio)
+    expect(parsed.album).to eq('wetransfer')
+    expect(parsed.artist).to eq('wetransfer')
+    expect(parsed.title).to eq('test')
+  end
+
+  it 'reads the mpeg frames correctly' do
+    fpath = fixtures_dir + '/MP3/test_read_frames.mp3'
+
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed.audio_sample_rate_hz). to eq(48000)
+  end
+
   it 'parses the Cassy MP3' do
     fpath = fixtures_dir + '/MP3/Cassy.mp3'
     parsed = subject.call(File.open(fpath, 'rb'))

data/spec/parsers/tiff_parser_spec.rb

@@ -47,12 +47,15 @@ describe FormatParser::TIFFParser do
     expect(parsed.intrinsics[:exif]).not_to be_nil
   end
 
-  it 'correctly extracts dimensions for a Sony ARW fixture' do
+  it 'parses Sony ARW fixture as arw format file' do
     arw_path = fixtures_dir + '/ARW/RAW_SONY_ILCE-7RM2.ARW'
 
     parsed = subject.call(File.open(arw_path, 'rb'))
 
     expect(parsed).not_to be_nil
+    expect(parsed.nature).to eq(:image)
+    expect(parsed.format).to eq(:arw)
+
     expect(parsed.width_px).to eq(7952)
     expect(parsed.height_px).to eq(5304)
     expect(parsed.intrinsics[:exif]).not_to be_nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.22.1
+  version: 0.24.2
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-08-26 00:00:00.000000000 Z
+date: 2020-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -34,7 +34,7 @@ dependencies:
         version: '1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.4
+        version: 1.3.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -44,27 +44,21 @@ dependencies:
         version: '1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.4
+        version: 1.3.7
 - !ruby/object:Gem::Dependency
   name: id3tag
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.10.1
+        version: '0.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.10.1
+        version: '0.14'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -202,6 +196,8 @@ files:
 - Rakefile
 - exe/format_parser_inspect
 - format_parser.gemspec
+- lib/active_storage/blob_analyzer.rb
+- lib/active_storage/blob_io.rb
 - lib/archive.rb
 - lib/attributes_json.rb
 - lib/audio.rb
@@ -241,6 +237,8 @@ files:
 - lib/read_limits_config.rb
 - lib/remote_io.rb
 - lib/video.rb
+- spec/active_storage/blob_io_spec.rb
+- spec/active_storage/rails_app_spec.rb
 - spec/attributes_json_spec.rb
 - spec/care_spec.rb
 - spec/esoteric_formats_spec.rb
@@ -248,6 +246,7 @@ files:
 - spec/format_parser_inspect_spec.rb
 - spec/format_parser_spec.rb
 - spec/hash_utils_spec.rb
+- spec/integration/active_storage/rails_app.rb
 - spec/io_utils_spec.rb
 - spec/parsers/aiff_parser_spec.rb
 - spec/parsers/bmp_parser_spec.rb