format_parser 0.23.0 → 0.25.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dddac3718ccf02324f4632adc68ad87d08ede00c3bbe64d85689dc8a6a06ad3a
-  data.tar.gz: 3ca9fb36416dffbd6fd1825f15ee0fe9b590633c959a165018f0c60f8d965361
+  metadata.gz: c24092543b6c98c713b07a45c3e3a2990332858397e11f6709470c32343a62f3
+  data.tar.gz: 0222b05ddfa1efa7cee364db7ad61ae7d44806b9622d6d6012fbb50e1b2e8138
 SHA512:
-  metadata.gz: 9193257f175a36087bc4f780659ce4e7bdefbbec456f38015b588ef9b378c71e5601c2d5de0f801297e6e968d70ed3e8db4a1fb0d8a2c78b90f8f9b328cfc43a
-  data.tar.gz: f181cbae4d8261e05ad3ed4787f9376c3738ba9487258b7d3885dcebe6166409578c82e2e5136c19b47b61ba4fc0d730f752b71a14e19cc6cea79ecdedc357e6
+  metadata.gz: 67a3a64166115ef70b2043c05e9c105c2939752b1de34086421a2b96311880673d1c668927eecf904e06db368cc9979218d666004b59a187f52e569f47e9d2a3
+  data.tar.gz: 5f4034dbcc2a92cb4908c6609465fbb81ec5011f42f22e9c5477b72a26ee4b59bbaa1191a13a5405d71f68ac8b8c0ceac652befd9eb2c1ea9fc5af000b221828

data/CHANGELOG.md

@@ -1,3 +1,20 @@
+## 0.25.0
+* MP3: add suport to id3 v2.4.x
+* JPEG: Update gem exifr to 1.3.8 to fix a bug
+
+## 0.24.2
+* Update gem id3tag to 0.14.0 to fix MP3 issues
+
+## 0.24.1
+* Fix MP3 frames reading to jump correctly to the next bytes
+
+## 0.24.0
+* The TIFF parser will now return :arw as format for Sony ARW files insted of :tif so that the caller can decide whether it
+  wants to deal with RAW processing or not
+
+## 0.23.1
+* Updated gem exifr to fix problems related to jpeg files from Olympos microscopes, which often have bad thumbnail data
+
 ## 0.23.0
 * Add ActiveStorage analyzer which can analyze ActiveStorage blobs. Enable it by setting
   `config.active_storage.analyzers.prepend FormatParser::ActiveStorage::BlobAnalyzer`

data/format_parser.gemspec

@@ -31,8 +31,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'ks', '~> 0.0'
-  spec.add_dependency 'exifr', '~> 1', '>= 1.3.4'
-  spec.add_dependency 'id3tag', '~> 0.13'
+  spec.add_dependency 'exifr', '~> 1', '>= 1.3.8'
+  spec.add_dependency 'id3tag', '~> 0.14'
   spec.add_dependency 'faraday', '~> 0.13'
   spec.add_dependency 'measurometer', '~> 1'
 

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.23.0'
+  VERSION = '0.25.0'
 end

data/lib/parsers/mp3_parser.rb

@@ -20,13 +20,14 @@ class FormatParser::MP3Parser
 
   # We limit the number of MPEG frames we scan
   # to obtain our duration estimation
-  MAX_FRAMES_TO_SCAN = 128
+  MAX_FRAMES_TO_SCAN = 500
 
   # Default frame size for mp3
   SAMPLES_PER_FRAME = 1152
 
   # For some edge cases
   ZIP_LOCAL_ENTRY_SIGNATURE = "PK\x03\x04\x14\x00".b
+  PNG_HEADER_BYTES = [137, 80, 78, 71, 13, 10, 26, 10].pack('C*')
 
   # Wraps the Tag object returned by ID3Tag in such
   # a way that a usable JSON representation gets
@@ -60,8 +61,12 @@ class FormatParser::MP3Parser
     # To avoid having that happen, we check for the PKZIP signature -
     # local entry header signature - at the very start of the file.
     # If the file is too small safe_read will fail too and the parser
-    # will terminate here.
-    return if safe_read(io, 6) == ZIP_LOCAL_ENTRY_SIGNATURE
+    # will terminate here. Same with PNGs. In the future
+    # we should implement "confidence" for MP3 as of all our formats
+    # it is by far the most lax.
+    header = safe_read(io, 8)
+    return if header.start_with?(ZIP_LOCAL_ENTRY_SIGNATURE)
+    return if header.start_with?(PNG_HEADER_BYTES)
 
     # Read all the ID3 tags (or at least attempt to)
     io.seek(0)
@@ -131,27 +136,28 @@ class FormatParser::MP3Parser
   # if you have a minute. https://pypi.python.org/pypi/tinytag
   def parse_mpeg_frames(io)
     mpeg_frames = []
+    bytes_to_read = 4
 
     MAX_FRAMES_TO_SCAN.times do |frame_i|
       # Read through until we can latch onto the 11 sync bits. Read in 4-byte
       # increments to save on read() calls
-      data = io.read(4)
+      data = io.read(bytes_to_read)
 
       # If we are at EOF - stop iterating
-      break unless data && data.bytesize == 4
+      break unless data && data.bytesize == bytes_to_read
 
       # Look for the sync pattern. It can be either the last byte being 0xFF,
       # or any of the 2 bytes in sequence being 0xFF and > 0xF0.
       four_bytes = data.unpack('C4')
       seek_jmp = sync_bytes_offset_in_4_byte_seq(four_bytes)
       if seek_jmp > 0
-        io.seek(io.pos + seek_jmp)
+        io.seek(io.pos - bytes_to_read + seek_jmp)
         next
       end
 
       # Once we are past that stage we have latched onto a sync frame header
       sync, conf, bitrate_freq, rest = four_bytes
-      frame_detail = parse_mpeg_frame_header(io.pos - 4, sync, conf, bitrate_freq, rest)
+      frame_detail = parse_mpeg_frame_header(io.pos - bytes_to_read, sync, conf, bitrate_freq, rest)
       mpeg_frames << frame_detail
 
       # There might be a xing header in the first frame that contains
@@ -166,7 +172,7 @@ class FormatParser::MP3Parser
         end
       end
       if frame_detail.frame_length > 1 # jump over current frame body
-        io.seek(io.pos + frame_detail.frame_length - 4)
+        io.seek(io.pos + frame_detail.frame_length - bytes_to_read)
       end
     end
     [nil, mpeg_frames]

data/lib/parsers/mp3_parser/id3_extraction.rb

@@ -1,6 +1,8 @@
 module FormatParser::MP3Parser::ID3Extraction
   ID3V1_TAG_SIZE_BYTES = 128
-  ID3V2_TAG_VERSIONS = ["\x43\x00".b, "\x03\x00".b, "\x02\x00".b]
+  # it supports 2.4.x, 2.3.x, 2.2.x which are supported by the gem id3tag
+  # see https://id3.org/Developer%20Information for more details of each version
+  ID3V2_MINOR_TAG_VERSIONS = [2, 3, 4]
   MAX_SIZE_FOR_ID3V2 = 1 * 1024 * 1024
 
   extend FormatParser::IOUtils
@@ -22,7 +24,7 @@ module FormatParser::MP3Parser::ID3Extraction
     io.seek(0) # Only support header ID3v2
     header = parse_id3_v2_header(io)
     return unless header[:tag] == 'ID3' && header[:size] > 0
-    return unless ID3V2_TAG_VERSIONS.include?(header[:version])
+    return unless ID3V2_MINOR_TAG_VERSIONS.include?(header[:version].unpack('C').first)
 
     id3_tag_size = io.pos + header[:size]
 

data/lib/parsers/tiff_parser.rb

@@ -26,7 +26,7 @@ class FormatParser::TIFFParser
     h = exif_data.height || exif_data.pixel_y_dimension
 
     FormatParser::Image.new(
-      format: :tif,
+      format: arw?(exif_data) ? :arw : :tif, # Specify format as arw for Sony ARW format images, else tif
       width_px: w,
       height_px: h,
       display_width_px: exif_data.rotated? ? h : w,
@@ -43,5 +43,11 @@ class FormatParser::TIFFParser
     safe_read(io, 2) == 'CR'
   end
 
+  # Similar to how exiftool determines the image type as ARW, we are implementing a check here
+  # https://github.com/exiftool/exiftool/blob/e969456372fbaf4b980fea8bb094d71033ac8bf7/lib/Image/ExifTool/Exif.pm#L929
+  def arw?(exif_data)
+    exif_data.compression == 6 && exif_data.new_subfile_type == 1 && exif_data.make == 'SONY'
+  end
+
   FormatParser.register_parser new, natures: :image, formats: :tif
 end

data/spec/parsers/mp3_parser_spec.rb

@@ -15,6 +15,12 @@ describe FormatParser::MP3Parser do
     expect(parsed.media_duration_seconds).to be_within(0.1).of(0.836)
   end
 
+  it 'does not misdetect a PNG' do
+    fpath = fixtures_dir + '/PNG/anim.png'
+    parsed = subject.call(File.open(fpath, 'rb'))
+    expect(parsed).to be_nil
+  end
+
   describe 'title/artist/album attributes' do
     let(:parsed) { subject.call(File.open(fpath, 'rb')) }
 
@@ -67,7 +73,7 @@ describe FormatParser::MP3Parser do
 
     large_syncsfe_size = [ID3Tag::SynchsafeInteger.encode(more_bytes_than_permitted)].pack('N')
     prepped = StringIO.new(
-      'ID3' + "\x43\x00".b + "\x00".b + large_syncsfe_size + gunk
+      'ID3' + "\x03\x00".b + "\x00".b + large_syncsfe_size + gunk
     )
 
     expect(ID3Tag).not_to receive(:read)
@@ -90,6 +96,14 @@ describe FormatParser::MP3Parser do
     expect(parsed.title).to eq('test')
   end
 
+  it 'reads the mpeg frames correctly' do
+    fpath = fixtures_dir + '/MP3/test_read_frames.mp3'
+
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed.audio_sample_rate_hz). to eq(48000)
+  end
+
   it 'parses the Cassy MP3' do
     fpath = fixtures_dir + '/MP3/Cassy.mp3'
     parsed = subject.call(File.open(fpath, 'rb'))
@@ -130,6 +144,14 @@ describe FormatParser::MP3Parser do
     }.to raise_error(FormatParser::IOUtils::InvalidRead)
   end
 
+  it 'supports id3 v2.4.x' do
+    fpath = fixtures_dir + '/MP3/id3v24.mp3'
+
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed.artist). to eq('wetransfer')
+  end
+
   describe '#as_json' do
     it 'converts all hash keys to string when stringify_keys: true' do
       fpath = fixtures_dir + '/MP3/Cassy.mp3'

data/spec/parsers/tiff_parser_spec.rb

@@ -47,12 +47,15 @@ describe FormatParser::TIFFParser do
     expect(parsed.intrinsics[:exif]).not_to be_nil
   end
 
-  it 'correctly extracts dimensions for a Sony ARW fixture' do
+  it 'parses Sony ARW fixture as arw format file' do
     arw_path = fixtures_dir + '/ARW/RAW_SONY_ILCE-7RM2.ARW'
 
     parsed = subject.call(File.open(arw_path, 'rb'))
 
     expect(parsed).not_to be_nil
+    expect(parsed.nature).to eq(:image)
+    expect(parsed.format).to eq(:arw)
+
     expect(parsed.width_px).to eq(7952)
     expect(parsed.height_px).to eq(5304)
     expect(parsed.intrinsics[:exif]).not_to be_nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.23.0
+  version: 0.25.0
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-08 00:00:00.000000000 Z
+date: 2020-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -34,7 +34,7 @@ dependencies:
         version: '1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.4
+        version: 1.3.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -44,21 +44,21 @@ dependencies:
         version: '1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.4
+        version: 1.3.8
 - !ruby/object:Gem::Dependency
   name: id3tag
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '0.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '0.14'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -292,7 +292,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A library for efficient parsing of file metadata