format_parser 1.3.0 → 1.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d90a6eee951feb5017bdedc0fb6dd4be49fc0a7f218972c0fb423b1985bf9a97
-  data.tar.gz: ee6caab359b0e01450897d00abd1f190b131e40d842d487d1f29107ac3b374f0
+  metadata.gz: 5a9a94233949cc72d18b433cf1ddcba0e479e8b93aa1ff2e48bda6f6d86f667b
+  data.tar.gz: 4e357bc46207e95cad52d21b2aaa1781e9c231bab02e235be29663db9722f5d9
 SHA512:
-  metadata.gz: 72aca621c20dfb24443e32c52a3d27f64a4887c1254f6d2295f77b39ce57e1a5ef0aa52f365630badcb2e3aa90d32c3546c8fcea3b83e025e2c4fc3606dd2dd7
-  data.tar.gz: 90da48352579c4044035732fd1f837d86db1a2d3c8f325b60ee688630f6cb4ea8efdb2ffd7887ac4e49cf9f9df2a22fe2189187c32d606b8f027a0a3d6a3ec5f
+  metadata.gz: dcf8c8aeefc6166f3645dae461aadbcc2b36e96cb7a75162586fc009d562f6f978767ff877b27d0c192b5ca3107011a1bfdda842e730e486ced02e4191b53f59
+  data.tar.gz: fbc2caafb269f5e9c249e6ffe62ea8141477589256fab1bea5d058877d571725b1e619619ef9fcd33005d42759697ae7c7575d7a79217bdedc7e96ab02ce3c1b

data/.gitignore

@@ -61,3 +61,6 @@ Gemfile.lock
 
 # rspec examples
 spec/examples.txt
+
+# IntelliJ config:
+/.idea/

data/CHANGELOG.md

@@ -1,6 +1,15 @@
+## 1.4.2
+* Fix `MP3Parser` taking precedence when parsing `WEBP` files.
+
+## 1.4.1
+* Skip Exif chunks that are malformed during `WEBP` parsing.
+
+## 1.4.0
+* Add support for `WEBP` lossy, lossless and extended file formats.
+
 ## 1.3.0
 * Add `heif_parser` and support for `HEIF` and `HEIC` formats. Exif parsing is still missing.
-* 
+
 ## 1.2.1
 * Resolve bug when `stts` atom is `nil`
 

data/README.md

@@ -33,6 +33,7 @@ and [dimensions,](https://github.com/sstephenson/dimensions) borrowing from them
 * OGG
 * MPEG, MPG
 * M3U
+* WEBP
 
 ...with [more](https://github.com/WeTransfer/format_parser/issues?q=is%3Aissue+is%3Aopen+label%3Aformats) on the way!
 
@@ -198,6 +199,10 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
 ### M3U
 - The M3U fixture files were created by one of the project maintainers
 
+### WEBP
+- With the exception of extended-animation.webp, which was obtained from Wikimedia Commons and is Creative Commons 
+licensed, all of the WebP fixture files have been created by one of the project maintainers.
+
 ### .key
 - The `keynote_recognized_as_jpeg.key` file was created by the project maintainers
 

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '1.3.0'
+  VERSION = '1.4.2'
 end

data/lib/io_utils.rb

@@ -42,6 +42,14 @@ module FormatParser::IOUtils
     safe_read(@buf, 4).unpack('N').first
   end
 
+  def read_little_endian_int_16
+    safe_read(@buf, 2).unpack('v').first
+  end
+
+  def read_little_endian_int_32
+    safe_read(@buf, 4).unpack('V').first
+  end
+
   # 'n' is the number of bytes to read
   def read_string(n)
     safe_read(@buf, n)

data/lib/parsers/mp3_parser.rb

@@ -28,6 +28,7 @@ class FormatParser::MP3Parser
   # For some edge cases
   ZIP_LOCAL_ENTRY_SIGNATURE = "PK\x03\x04\x14\x00".b
   PNG_HEADER_BYTES = [137, 80, 78, 71, 13, 10, 26, 10].pack('C*')
+  WEBP_HEADER_REGEX = /RIFF.{4}WEBP/i
 
   MAGIC_LE = [0x49, 0x49, 0x2A, 0x0].pack('C4')
   MAGIC_BE = [0x4D, 0x4D, 0x0, 0x2A].pack('C4')
@@ -68,9 +69,10 @@ class FormatParser::MP3Parser
     # will terminate here. Same with PNGs. In the future
     # we should implement "confidence" for MP3 as of all our formats
     # it is by far the most lax.
-    header = safe_read(io, 8)
+    header = safe_read(io, 12)
     return if header.start_with?(ZIP_LOCAL_ENTRY_SIGNATURE)
     return if header.start_with?(PNG_HEADER_BYTES)
+    return if header.start_with?(WEBP_HEADER_REGEX)
 
     io.seek(0)
     return if TIFF_HEADER_BYTES.include?(safe_read(io, 4))

data/lib/parsers/webp_parser.rb

@@ -0,0 +1,169 @@
+# WebP is an image format that provides superior lossless and lossy compression for images on the web, with support for
+# transparency. It uses predictive coding to encode an image, predicting the values in a block of pixels based on the
+# values of neighbouring blocks. A WebP file consists of VP8 or VP8L data, and a container based on RIFF. There is also
+# an extended file format, VP8X, that optionally encodes various information such as the color profile, animation
+# control data, transparency, and EXIF and/or XMP metadata.
+#
+# For more information, visit https://developers.google.com/speed/webp.
+#
+# TODO: Decide how to determine color mode (depends on variant, transformations, flags, etc.; maybe not worth it).
+
+class FormatParser::WebpParser
+  include FormatParser::EXIFParser
+  include FormatParser::IOUtils
+
+  WEBP_MIME_TYPE = 'image/webp'
+
+  def likely_match?(filename)
+    filename =~ /\.webp$/i
+  end
+
+  def call(io)
+    @buf = FormatParser::IOConstraint.new(io)
+
+    # All WebP files start with the following 20 bytes:
+    #
+    # Offset  | Description
+    # -------------------------------------------------------------------------------------
+    # 0...3   | "RIFF" (Since WebP is based on the RIFF file container format).
+    # 4...7   | The size of the file in bytes - 8 bytes.
+    # 8...11  | "WEBP" (To signify that this is a WebP file).
+    # 12...15 | The VB8 variant in use ("VB8 ", "VP8L" or "VB8X")
+    # 16...19 | The length of the VB8 data in bytes (i.e. The size of the file - 20 bytes).
+    riff, webp, variant = safe_read(@buf, 20).unpack('A4x4A4A4')
+    return unless riff == 'RIFF' && webp == 'WEBP'
+    read_data(variant)
+  end
+
+  private
+
+  def read_data(variant)
+    case variant
+    when 'VP8' # Lossy
+      read_lossy_data
+    when 'VP8L' # Lossless
+      read_lossless_data
+    when 'VP8X' # Extended
+      read_extended_data
+    else
+      nil
+    end
+  end
+
+  def read_lossy_data
+    # Encoded as a single VP8 key frame - a 10-byte uncompressed chunk followed by 2+ partitions of compressed data.
+    # The first 6 bytes of this chunk contains information that is mostly relevant when using VP8 as a video
+    # compression format, and can be ignored.
+    safe_skip(@buf, 6)
+
+    # The subsequent 4 bytes contain the image width and height, respectively, as 16-bit unsigned little endian
+    # integers.
+    width, height = safe_read(@buf, 4).unpack('S<S<')
+    create_image(width, height)
+  end
+
+  def read_lossless_data
+    # There is a single byte signature, 0x2F, that we can disregard.
+    safe_skip(@buf, 1)
+
+    # The subsequent 4 bytes contain the image width and height, respectively, as 14-bit unsigned little endian
+    # integers (minus one). The 4 remaining bits consist of a 1-bit flag indicating whether alpha is used, and a 3-bit
+    # version that is always zero.
+    dimensions = read_little_endian_int_32
+    width = (dimensions & 0x3fff) + 1
+    height = (dimensions >> 14 & 0x3fff) + 1
+    has_transparency = (dimensions >> 28 & 0x1) == 1
+
+    create_image(width, height, has_transparency: has_transparency)
+  end
+
+  def read_extended_data
+    # After the common RIFF header bytes, the extended file format has a series of 1-bit flags to signify the presence
+    # of optional information. These flags are as follows:
+    #
+    # |0|1|2|3|4|5|6|7|
+    # +-+-+-+-+-+-+-+-+
+    # |Rsv|I|L|E|X|A|R|
+    #
+    # Where:
+    #   - Rsv & R = Reserved - Should be 0.
+    #   - I = Set if file contains an ICC profile.
+    #   - L = Set if file contains transparency information.
+    #   - E = Set if file contains Exif metadata.
+    #   - X = Set if file contains XMP metadata.
+    #   - A = Set if file is an animated image.
+    flags = read_int_8
+    has_transparency = flags & 0x10 != 0
+    has_exif_metadata = flags & 0x08 != 0
+    has_xmp_metadata = flags & 0x04 != 0
+    has_multiple_frames = flags & 0x02 != 0
+
+    # The flags are followed by three reserved bytes of zeros, and then by the width and height, respectively - each
+    # occupying three bytes and each one less than the actual canvas measurements.
+    safe_skip(@buf, 3)
+    dimensions = safe_read(@buf, 6).unpack('VS')
+    width = (dimensions[0] & 0xffffff) + 1
+    height = (dimensions[0] >> 24 | dimensions[1] << 8 & 0xffffff) + 1
+
+    image = create_image(width, height, has_multiple_frames: has_multiple_frames, has_transparency: has_transparency)
+    augment_image(image) if has_exif_metadata || has_xmp_metadata || has_multiple_frames
+    image
+  end
+
+  def create_image(width, height, has_multiple_frames: false, has_transparency: false)
+    FormatParser::Image.new(
+      content_type: WEBP_MIME_TYPE,
+      format: :webp,
+      has_multiple_frames: has_multiple_frames,
+      has_transparency: has_transparency,
+      height_px: height,
+      width_px: width
+    )
+  end
+
+  def augment_image(image)
+    # We're going to scan the file looking for the EXIF, XMP and/or ANMF chunks.
+    intrinsics = {}
+    num_frames = 0
+    loop do
+      # Try to read the next chunk header, and break the loop if we've reached EOF.
+      begin
+        fourcc, chunk_size = safe_read(@buf, 8).unpack('A4V')
+      rescue FormatParser::IOUtils::InvalidRead
+        break
+      end
+
+      # Padding byte of 0 added if chunk size is odd.
+      safe_skip(@buf, 1) if chunk_size.odd?
+
+      case fourcc
+      when 'EXIF'
+        chunk_pos = @buf.pos
+        begin
+          exif = exif_from_tiff_io(StringIO.new(safe_read(@buf, chunk_size)))
+          # We use ||= here as one Exif chunk at most should be present, even though it is possible for there to be more.
+          intrinsics[:exif] ||= exif
+          image.height_px, image.width_px = image.width_px, image.height_px if exif&.rotated?
+          image.orientation = exif&.orientation_sym
+        rescue EXIFR::MalformedTIFF
+          # Exif data was malformed and could not be parsed. Need to ensure that buffer is pointing at the next chunk.
+          @buf.seek(chunk_pos + chunk_size)
+          next
+        end
+      when 'XMP'
+        # We use ||= here as one XMP chunk at most should be present, even though it is possible for there to be more.
+        intrinsics[:xmp] ||= safe_read(@buf, chunk_size)
+      when 'ANMF'
+        num_frames += 1 if image.has_multiple_frames
+        safe_skip(@buf, chunk_size)
+      else
+        safe_skip(@buf, chunk_size)
+      end
+    end
+
+    image.intrinsics = intrinsics unless intrinsics.empty?
+    image.num_animation_or_video_frames = num_frames if num_frames > 0
+  end
+
+  FormatParser.register_parser new, natures: [:image], formats: [:webp]
+end

data/spec/parsers/webp_parser_spec.rb

@@ -0,0 +1,136 @@
+require 'spec_helper'
+
+describe FormatParser::WebpParser do
+  it 'does not parse files with an invalid RIFF header' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/invalid-header.webp', 'rb'))
+    expect(result).to be_nil
+  end
+
+  it 'does not parse files with an unrecognised variant' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/unrecognised-variant.webp', 'rb'))
+    expect(result).to be_nil
+  end
+
+  it 'successfully parses lossy (VP8) WebP files' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/lossy.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(false)
+    expect(result.has_transparency).to eq(false)
+    expect(result.height_px).to eq(181)
+    expect(result.intrinsics).to be_nil
+    expect(result.orientation).to be_nil
+    expect(result.width_px).to eq(65)
+  end
+
+  it 'successfully parses lossless WebP files' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/lossless.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(false)
+    expect(result.has_transparency).to eq(false)
+    expect(result.height_px).to eq(181)
+    expect(result.intrinsics).to be_nil
+    expect(result.orientation).to be_nil
+    expect(result.width_px).to eq(65)
+  end
+
+  it 'successfully parses lossless WebP files with an alpha channel' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/lossless-alpha.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(false)
+    expect(result.has_transparency).to eq(true)
+    expect(result.height_px).to eq(181)
+    expect(result.intrinsics).to be_nil
+    expect(result.orientation).to be_nil
+    expect(result.width_px).to eq(65)
+  end
+
+  it 'successfully parses extended WebP files' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/extended.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(false)
+    expect(result.has_transparency).to eq(false)
+    expect(result.height_px).to eq(181)
+    expect(result.intrinsics).to be_nil
+    expect(result.orientation).to be_nil
+    expect(result.width_px).to eq(65)
+  end
+
+  it 'successfully parses extended WebP files with an alpha channel' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/extended-alpha.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(false)
+    expect(result.has_transparency).to eq(true)
+    expect(result.height_px).to eq(181)
+    expect(result.intrinsics).to be_nil
+    expect(result.orientation).to be_nil
+    expect(result.width_px).to eq(65)
+  end
+
+  it 'successfully parses extended WebP files with Exif metadata' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/extended-exif.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(false)
+    expect(result.has_transparency).to eq(false)
+    expect(result.height_px).to eq(181)
+    expect(result.intrinsics).not_to be_nil
+    expect(result.intrinsics[:exif]).not_to be_nil
+    expect(result.intrinsics[:exif].image_length).to eq(result.height_px)
+    expect(result.intrinsics[:exif].image_width).to eq(result.width_px)
+    expect(result.orientation).to eq(:top_left)
+    expect(result.width_px).to eq(65)
+  end
+
+  it 'successfully parses extended WebP files with XMP metadata' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/extended-xmp.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(false)
+    expect(result.has_transparency).to eq(false)
+    expect(result.height_px).to eq(181)
+    expect(result.intrinsics).not_to be_nil
+    expect(result.intrinsics[:xmp]).not_to be_nil
+    expect(result.orientation).to be_nil
+    expect(result.width_px).to eq(65)
+  end
+
+  it 'successfully parses extended WebP files with animation' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/extended-animation.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(true)
+    expect(result.has_transparency).to eq(true)
+    expect(result.height_px).to eq(211)
+    expect(result.intrinsics).to be_nil
+    expect(result.orientation).to be_nil
+    expect(result.width_px).to eq(211)
+  end
+
+  it 'successfully skips malformed Exif chunks' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/extended-malformed-exif.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(false)
+    expect(result.has_transparency).to eq(false)
+    expect(result.height_px).to eq(181)
+    expect(result.intrinsics).not_to be_nil
+    expect(result.intrinsics[:exif]).to be_nil
+    expect(result.intrinsics[:xmp]).not_to be_nil
+    expect(result.orientation).to be_nil
+    expect(result.width_px).to eq(65)
+  end
+end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.2
 platform: ruby
 authors:
 - Noah Berman
 - Julik Tarkhanov
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-05-31 00:00:00.000000000 Z
+date: 2022-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -252,6 +252,7 @@ files:
 - lib/parsers/psd_parser.rb
 - lib/parsers/tiff_parser.rb
 - lib/parsers/wav_parser.rb
+- lib/parsers/webp_parser.rb
 - lib/parsers/zip_parser.rb
 - lib/parsers/zip_parser/file_reader.rb
 - lib/parsers/zip_parser/office_formats.rb
@@ -291,6 +292,7 @@ files:
 - spec/parsers/psd_parser_spec.rb
 - spec/parsers/tiff_parser_spec.rb
 - spec/parsers/wav_parser_spec.rb
+- spec/parsers/webp_parser_spec.rb
 - spec/parsers/zip_parser_spec.rb
 - spec/read_limiter_spec.rb
 - spec/read_limits_config_spec.rb
@@ -302,7 +304,7 @@ licenses:
 - MIT (Hippocratic)
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -317,8 +319,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
-signing_key: 
+rubygems_version: 3.3.7
+signing_key:
 specification_version: 4
 summary: A library for efficient parsing of file metadata
 test_files: []