format_parser 1.2.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8bdd6daa59e43bbe6033562b804d3b1b9685c6e2aa5ef3f8d527c74516e1f6c
-  data.tar.gz: d6035f0b3c819085ffd23c4988e48145976ba1f98547d16589b2a4b4aa3e7aa7
+  metadata.gz: cf7fbbf842a1ae6fcde3986b360877223ac699a87950848b508da15f8a8280ad
+  data.tar.gz: 29882db7afe75a1d3b6554f18dbc837cefb1dbe9e8927adafe959ac8d37ade84
 SHA512:
-  metadata.gz: 1a2f6243d295589972c63e45b719f19a5761c872103c32bfaacaef9cee9b85551c18ddaaf3c4095567c358958e7b7bf78800fc3e5a8e6c9f774eab383dc3160c
-  data.tar.gz: 73ba4f1099ccb4c490b50a8f531be843679159a6417980b2a61b905724905cf6b7e884127b66096e2237982dc6743e7f057bea0062472c1779e4d754dd5388ff
+  metadata.gz: 0cf33f73ac298f565020e9819c9c7d2e2af340490b869b97a008b4466ac2b0825fed70d5d9e255ef1192520cef92fdeeff0c7ade18d5e38910d6dc2fd0de89f3
+  data.tar.gz: c20cdc92df0d29d1e0c4b9f8c05644e17216f239a8d90e9c7af38f5566b4abaaf6f6289d5cc69d6a25b0ed644236403820b5c3402080f0b7ba40ca112b671d3a

data/.gitignore

@@ -61,3 +61,6 @@ Gemfile.lock
 
 # rspec examples
 spec/examples.txt
+
+# IntelliJ config:
+/.idea/

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 1.4.0
+* Add support for `WEBP` lossy, lossless and extended file formats.
+
+## 1.3.0
+* Add `heif_parser` and support for `HEIF` and `HEIC` formats. Exif parsing is still missing.
+
+## 1.2.1
+* Resolve bug when `stts` atom is `nil`
+
 ## 1.2.0
 * Add support for `codecs` in moov_parser for video metadata
 

data/Gemfile

@@ -1,9 +1,4 @@
 source 'https://rubygems.org'
 
-gem 'ruby-debug-ide'
-gem 'debase'
-gem 'solargraph', group: :development
-gem 'pry', group: :development
-
 # Gem dependencies specified in the gemspec
 gemspec

data/README.md

@@ -33,6 +33,7 @@ and [dimensions,](https://github.com/sstephenson/dimensions) borrowing from them
 * OGG
 * MPEG, MPG
 * M3U
+* WEBP
 
 ...with [more](https://github.com/WeTransfer/format_parser/issues?q=is%3Aissue+is%3Aopen+label%3Aformats) on the way!
 
@@ -198,6 +199,10 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
 ### M3U
 - The M3U fixture files were created by one of the project maintainers
 
+### WEBP
+- With the exception of extended-animation.webp, which was obtained from Wikimedia Commons and is Creative Commons 
+licensed, all of the WebP fixture files have been created by one of the project maintainers.
+
 ### .key
 - The `keynote_recognized_as_jpeg.key` file was created by the project maintainers
 

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '1.2.0'
+  VERSION = '1.4.0'
 end

data/lib/io_utils.rb

@@ -30,5 +30,30 @@ module FormatParser::IOUtils
     nil
   end
 
+  def read_int_8
+    safe_read(@buf, 1).unpack('C').first
+  end
+
+  def read_int_16
+    safe_read(@buf, 2).unpack('n').first
+  end
+
+  def read_int_32
+    safe_read(@buf, 4).unpack('N').first
+  end
+
+  def read_little_endian_int_16
+    safe_read(@buf, 2).unpack('v').first
+  end
+
+  def read_little_endian_int_32
+    safe_read(@buf, 4).unpack('V').first
+  end
+
+  # 'n' is the number of bytes to read
+  def read_string(n)
+    safe_read(@buf, n)
+  end
+
   ### TODO: Some kind of built-in offset for the read
 end

data/lib/parsers/heif_parser.rb

@@ -0,0 +1,431 @@
+# HEIF stands for High-Efficiency Image File format, which is basically a container that is capable of storing an image, or a sequence of images in a single file.
+# There are a number of variants of HEIF, which can be used to store images, sequences of images, or videos using different codecs.
+# The variant that Apple uses to store images and sequences of images in its iOS and macOS operating systems is High Efficiency Image Coding (HEIC), which uses HEVC / H.265 for content compression.
+class FormatParser::HEIFParser
+  include FormatParser::IOUtils
+
+  HEIF_MARKER = [0x68, 0x65, 0x69, 0x63].pack('C4') # heif marker
+  FILE_TYPE_BOX_MARKER = [0x66, 0x74, 0x79, 0x70].pack('C4') # ftyp marker
+  META_BOX_MARKER = [0x6D, 0x65, 0x74, 0x61].pack('C4') # meta marker
+  MIF1_MARKER = [0x6D, 0x69, 0x66, 0x31].pack('C4') # mif1 marker
+  MSF1_MARKER = [0x6D, 0x73, 0x66, 0x31].pack('C4') # msf1 marker
+  MEANINGLESS_BYTE = [0x00, 0x00, 0x00, 0x00].pack('C4')
+  HANDLER_MARKER = [0x68, 0x64, 0x6C, 0x72].pack('C4') # hdlr marker
+  ITEM_PROPERTIES_BOX = [0x69, 0x70, 0x72, 0x70].pack('C4') # iprp marker
+  ITEM_PROPERTIES_CONTAINER_BOX = [0x69, 0x70, 0x63, 0x6F].pack('C4') # ipco marker
+  IMAGE_SPATIAL_EXTENTS_BOX = [0x69, 0x73, 0x70, 0x65].pack('C4') # ispe marker
+  PIXEL_ASPECT_RATIO_BOX = [0x70, 0x61, 0x73, 0x70].pack('C4') # pasp marker
+  ITEM_INFO_BOX = [0x69, 0x69, 0x6E, 0x66].pack('C4') # iinf marker
+  ITEM_INFO_ENTRY = [0x69, 0x6E, 0x66, 0x65].pack('C4') # infe marker
+  MIME_MARKER = [0x6D, 0x69, 0x6D, 0x65].pack('C4') # mime marker
+  COLOUR_INFO_BOX = [0x63, 0x6F, 0x6C, 0x72].pack('C4') # colr marker
+  PIXEL_INFO_BOX = [0x70, 0x69, 0x78, 0x69].pack('C4') # pixi marker
+  RELATIVE_LOCATION_BOX = [0x72, 0x6C, 0x6F, 0x63].pack('C4') # rloc marker
+  CLEAN_APERTURE_BOX = [0x63, 0x6C, 0x61, 0x70].pack('C4') # clap marker
+  PRIMARY_ITEM_BOX = [0x70, 0x69, 0x74, 0x6D].pack('C4') # pitm marker
+  ITEM_PROPERTIES_ASSOCIATION_BOX = [0x69, 0x70, 0x6D, 0x61].pack('C4') # ipma marker
+  IMAGE_ROTATION_BOX = [0x69, 0x72, 0x6F, 0x74].pack('C4') # irot marker
+  HEADER_LENGTH = 8 # every box header has a length of 8 bytes
+  HEIC_MIME_POSSIBLE_TYPES = {
+    'heic' => :heic,
+    'heix' => :heix,
+    'heim' => :heim,
+    'heis' => :heis
+  }
+  HEIC_MIME_TYPE = 'image/heic'
+  HEIF_MIME_TYPE = 'image/heif'
+  # TODO: use the following when adding image-sequence parsing
+  # HEIC_SEQUENCE_MIME_TYPE = 'image/heic-sequence'
+  # HEIF_SEQUENCE_MIME_TYPE = 'image/heif-sequence'
+
+  def self.call(io)
+    new.call(io)
+  end
+
+  def call(io)
+    @buf                = FormatParser::IOConstraint.new(io)
+    @format             = nil
+    @@major_brand       = nil
+    @width              = nil
+    @height             = nil
+    @exif_data_frames   = []
+    @compatible_brands  = nil
+    @metadata_start_pos = 0
+    @metadata_end_pos   = 0
+    @handler_type       = nil
+    @sub_items          = nil
+    @pixel_aspect_ratio = nil
+    @colour_info        = nil
+    @pixel_info         = nil
+    @horizontal_offset  = nil
+    @vertical_offset    = nil
+    @clean_aperture     = nil
+    @primary_item_id    = 0
+    @item_props         = {}
+    @rotation           = 0
+    @item_props_idxs    = []
+    @content_type       = nil
+    scan
+  end
+
+  def scan
+    # All HEIC files must be conform to ISO/IEC 23008-12:2017
+    # Moreover, all HEIC files are conform to ISO/IEC 14496-12:2015 and should be conform to the Clause 4 of such spec.
+    # Files are formed as a series of objects, called boxes. All data is contained in such boxes.
+    # All boxes start with a header which defines both size and type.
+    # The size is the entire size of the box, including the size and type header, fields, and all contained boxes.
+    # The fields in the objects are stored with the most significant byte first, commonly known as network byte order or big-endian format.
+    # A HEIC file must contain a File Type Box (ftyp).
+    # A file conforms to all the requirements of the brands listed in the compatible_brands.
+    scan_file_type_box
+
+    # file may be identified by MIME type of Annex C of ISO/IEC 23008-12 if 'mif1' is the major brand or Annex D if 'msf1' is the major brand.
+    # the MIME indicates the nature and format of our assortment of bytes
+    # note particularly that the brand 'mif1' doesn't mandate a MovieBox ("moov").
+    # One or more brands must be included in the list of compatible brands
+    return if @compatible_brands.nil?
+    if @compatible_brands&.include?(MIF1_MARKER)
+      scan_meta_level_box
+      if @major_brand == MIF1_MARKER
+        @content_type = HEIF_MIME_TYPE
+        @format = :heif
+      elsif (@compatible_brands & HEIC_MIME_POSSIBLE_TYPES.keys).length > 0
+        @format = :heic
+        @content_type = HEIC_MIME_TYPE
+      end
+    end
+    if @compatible_brands&.include?(MSF1_MARKER)
+      # TODO
+    end
+
+    result = FormatParser::Image.new(
+      format: @format,
+      width_px: @width,
+      height_px: @height,
+      intrinsics: {
+        compatible_brands: @compatible_brands,
+        handler_type: @handler_type,
+        # 'sub_items': @sub_items, # enable this if you want to output all the sub-items in the image
+        pixel_aspect_ratio: @pixel_aspect_ratio,
+        colour_info: @colour_info,
+        pixel_info: @pixel_info,
+        horizontal_offset: @horizontal_offset,
+        vertical_offset: @vertical_offset,
+        clean_aperture: @clean_aperture,
+        rotation: @rotation
+      },
+      content_type: @content_type
+    )
+
+    result
+  end
+
+  def scan_file_type_box
+    file_type_box_length = read_int_32
+    return unless read_string(4) == FILE_TYPE_BOX_MARKER
+    @major_brand = read_string(4)
+    return unless @major_brand == HEIF_MARKER || @major_brand == MIF1_MARKER
+    read_string(4) # minor_brand
+
+    # Subtracting from the total length of the box specified in the header the size header itself (8 bytes = header length and length of ftyp)
+    # and the length of the major and minor brand, we obtain the compatible brands
+    data_left_length = file_type_box_length - HEADER_LENGTH - HEIF_MARKER.length - 4
+
+    @compatible_brands = []
+    (data_left_length / 4).times do
+      @compatible_brands << read_string(4)
+    end
+  end
+
+  def scan_meta_level_box
+    metadata_length = read_int_32
+    return unless read_string(4) == META_BOX_MARKER
+    @metadata_start_pos = @buf.pos
+    @metadata_end_pos = @buf.pos + metadata_length - HEADER_LENGTH # the real data is always without the 8 initial bytes of the handler
+    read_nil_version_and_flag
+
+    # we are looking for box/containers right beneath the Meta box
+    # we start with the HDLR (Handler) box..
+    handler_length = read_int_32
+    return unless read_string(4) == HANDLER_MARKER
+    handler_length -= HEADER_LENGTH # subtract the header as usual (will not be mentioned anymore from now on)
+    handler_start = @buf.pos
+    # the handler type declares the type of metadata and thus the process by which the media-data in the track is presented
+    # it also indicates the structure or format of the ‘meta’ box contents
+    read_nil_version_and_flag
+    read_string(4) # pre_defined bytes, always 4 null bytes in the hdlr box
+    @handler_type = read_string(4)
+    @buf.seek(handler_start + handler_length) # the remaining part is reserved
+
+    # ..continue looking for the IINF box and especially for the IPRP box, containing info about the image itself
+    next_box_length = read_int_32
+    next_box = read_string(4)
+    next_box_start_pos = @buf.pos
+    while @buf.pos < @metadata_end_pos # we iterate over all next incoming boxed but without going outside the meta-box
+      case next_box
+      when PRIMARY_ITEM_BOX
+        read_primary_item_box
+      when ITEM_INFO_BOX
+        read_item_info_box
+      when ITEM_PROPERTIES_BOX
+        read_item_properties_box
+        fill_primary_values
+      when next_box == ''
+        break
+      end
+      next_box_length, next_box, next_box_start_pos = get_next_box(next_box_start_pos, next_box_length, @metadata_end_pos)
+    end
+  end
+
+  def read_item_info_box
+    version = read_int_8
+    safe_skip(@buf, 3) # 0 flags
+    entry_count = if version == 0
+      read_int_16
+    else
+      read_int_32
+    end
+    @sub_items = []
+    entry_count.times {
+      item_info_entry_length = read_int_32
+      return unless read_string(4) == ITEM_INFO_ENTRY
+      item_info_end_pos = @buf.pos + item_info_entry_length - HEADER_LENGTH
+      version = read_int_8
+      safe_skip(@buf, 3) # 0 flags
+      case version
+      when 2
+        item_id = read_int_16
+      when 3
+        item_id = read_int_32
+      else
+        return # wrong version according to standards, hence return
+      end
+      safe_skip(@buf, 2) # not interested in the item_protection_index
+      item_type = read_string(4)
+      content_encoding = ''
+      if item_type == MIME_MARKER
+        content_encoding = read_string(item_info_end_pos - @buf.pos).delete!("\0") # remove the null-termination part for output visualization reason
+      end
+      @sub_items << {item_id: item_id, item_type: item_type, content_encoding: content_encoding}
+      @buf.seek(item_info_end_pos) # we are not interested in anything else, go directly to the end of this 'infe' box
+    }
+  end
+
+  def read_nil_version_and_flag
+    safe_skip(@buf, 1) # version, always 0 in this current box
+    safe_skip(@buf, 3) # flags, always 0 in this current box
+  end
+
+  def read_primary_item_box
+    version = read_int_8
+    safe_read(@buf, 3) # flags, always 0 in this current box
+    @primary_item_id = if version == 0
+      read_int_16
+    else
+      read_int_32
+    end
+  end
+
+  # the ITEM_PROPERTIES_CONTAINER_BOX contains an implicitely 1-based index list of item properties.
+  # While parsing such box we are storing the properties with its own index.
+  # Reason behind is that the primary_item will be associated to some of these properties through the same index
+  # and in order to output relevant data from the format_parser we need all the properties associated to the primary_item.
+  # Hence the need of the association between an item and its properties, found in the ITEM_PROPERTIES_ASSOCIATION_BOX
+  def read_item_properties_box
+    ipco_length = read_int_32
+    return unless read_string(4) == ITEM_PROPERTIES_CONTAINER_BOX
+    read_item_properties_container_box(ipco_length)
+    read_int_32 # ipma_length
+    return unless read_string(4) == ITEM_PROPERTIES_ASSOCIATION_BOX
+    read_item_properties_association_box
+  end
+
+  def read_item_properties_container_box(box_length)
+    end_of_ipco_box = @buf.pos + box_length - HEADER_LENGTH
+    item_prop_length = read_int_32
+    item_prop_name = read_string(4)
+    item_prop_start_pos = @buf.pos
+    item_prop_index = 1
+    while @buf.pos < end_of_ipco_box
+      case item_prop_name
+      when IMAGE_SPATIAL_EXTENTS_BOX
+        read_nil_version_and_flag
+        width = read_int_32
+        height = read_int_32
+        @item_props[item_prop_index] = {
+          type: IMAGE_SPATIAL_EXTENTS_BOX,
+          width: width,
+          height: height
+        }
+      when PIXEL_ASPECT_RATIO_BOX
+        h_spacing = read_int_32
+        v_spacing = read_int_32
+        pixel_aspect_ratio = "#{h_spacing}/#{v_spacing}"
+        @item_props[item_prop_index] = {
+          type: PIXEL_ASPECT_RATIO_BOX,
+          pixel_aspect_ratio: pixel_aspect_ratio
+        }
+      when COLOUR_INFO_BOX
+        colour_info = {
+          colour_primaries: read_int_16,
+          transfer_characteristics: read_int_16,
+          matrix_coefficients: read_int_16
+        }
+        @item_props[item_prop_index] = {
+          type: COLOUR_INFO_BOX,
+          colour_info: colour_info
+        }
+      when PIXEL_INFO_BOX
+        pixel_info = []
+        read_nil_version_and_flag
+        num_channels = read_int_8
+        channel = 1
+        while channel <= num_channels
+          channel += 1
+          pixel_info << {
+            "bits_in_channel_#{channel}": read_int_8
+          }
+        end
+        @item_props[item_prop_index] = {
+          type: PIXEL_INFO_BOX,
+          pixel_info: pixel_info
+        }
+      when RELATIVE_LOCATION_BOX
+        read_nil_version_and_flag
+        horizontal_offset = read_int_32
+        vertical_offset = read_int_32
+        @item_props[item_prop_index] = {
+          type: RELATIVE_LOCATION_BOX,
+          horizontal_offset: horizontal_offset,
+          vertical_offset: vertical_offset
+        }
+      when CLEAN_APERTURE_BOX
+        clean_aperture = []
+        clean_aperture << {
+          clean_aperture_width_n: read_int_32,
+          clean_aperture_width_d: read_int_32,
+          clean_aperture_height_n: read_int_32,
+          clean_aperture_height_d: read_int_32,
+          horiz_off_n: read_int_32,
+          horiz_off_d: read_int_32,
+          vert_off_n: read_int_32,
+          vert_off_d: read_int_32
+        }
+        @item_props[item_prop_index] = {
+          type: CLEAN_APERTURE_BOX,
+          clean_aperture: clean_aperture
+        }
+      when IMAGE_ROTATION_BOX
+        read_nil_version_and_flag
+        binary = convert_byte_to_binary(read_int_8)
+        # we need only the last 2 bits to retrieve the angle multiplier. angle multiplier * 90 specifies the angle
+        rotation = binary.slice(6, 2).join.to_i(2) * 90
+        @item_props[item_prop_index] = {
+          type: IMAGE_ROTATION_BOX,
+          rotation: rotation
+        }
+      end
+      item_prop_length, item_prop_name, item_prop_start_pos = get_next_box(item_prop_start_pos, item_prop_length, end_of_ipco_box)
+      item_prop_index += 1
+    end
+  end
+
+  def read_item_properties_association_box
+    version = read_int_8
+    safe_read(@buf, 2) # we skip the first 2 bytes of the flags (total of 3 bytes) cause we care only about the least significant bit
+    flags = read_int_8
+    entry_count = read_int_32
+    item_id = 0
+    entry_count.times do
+      item_id = if version == 0
+        read_int_16
+      else
+        read_int_32
+      end
+
+      association_count = read_int_8
+      association_count.times do
+        # we need to retrieve the "essential" bit wich is just the first bit in the next byte
+        binary = convert_byte_to_binary(read_int_8)
+        # essential_bit = binary[0] # uncomment if needed
+        binary.concat(convert_byte_to_binary(read_int_8)) if (flags & 1) == 1 # if flag is 1 we need the next 15 bits instead of only the next 7 bits
+        # we need to nullify the 1st bit since that one was the essential bit and doesn't count now to calculate the property index
+        binary[0] = 0
+        item_property_index = binary.join.to_i(2)
+        # we are interested only in the primary item properties
+        @item_props_idxs << item_property_index if item_id == @primary_item_id
+      end
+
+      # we are interested only in the primary item
+      if item_id != @primary_item_id
+        next
+      else
+        return
+      end
+    end
+  end
+
+  def fill_primary_values
+    @item_props_idxs.each { |x|
+      next if @item_props[x].nil?
+      prop = @item_props[x]
+      case prop[:type]
+      when IMAGE_SPATIAL_EXTENTS_BOX
+        @width = prop[:width]
+        @height = prop[:height]
+      when PIXEL_ASPECT_RATIO_BOX
+        @pixel_aspect_ratio = prop[:pixel_aspect_ratio]
+      when COLOUR_INFO_BOX
+        @colour_info = prop[:colour_info]
+      when PIXEL_INFO_BOX
+        @pixel_info = prop[:pixel_info]
+      when RELATIVE_LOCATION_BOX
+        @horizontal_offset = prop[:horizontal_offset]
+        @vertical_offset = prop[:vertical_offset]
+      when CLEAN_APERTURE_BOX
+        @clean_aperture = prop[:clean_aperture]
+      when IMAGE_ROTATION_BOX
+        @rotation = prop[:rotation]
+      end
+    }
+  end
+
+  def next_meaningful_meta_byte
+    while @buf.pos < @metadata_end_pos
+      next_byte = read_string(4)
+      return next_byte if meaningful?(next_byte)
+    end
+  end
+
+  def get_next_box(box_start_pos, box_length, end_pos_upper_box)
+    skip_pos = box_start_pos + box_length - HEADER_LENGTH
+    @buf.seek(skip_pos)
+    return if skip_pos >= end_pos_upper_box
+    next_box_length = read_int_32
+    next_box_name = read_string(4)
+    [next_box_length, next_box_name, @buf.pos]
+  end
+
+  def meaningful?(byte)
+    byte != MEANINGLESS_BYTE
+  end
+
+  def convert_byte_to_binary(integer)
+    binary = []
+    while integer > 0
+      binary << integer % 2
+      integer /= 2
+    end
+    binary_value = binary.reverse
+    (8 - binary_value.length).times do
+      binary_value.prepend('0')
+    end
+    binary_value
+  end
+
+  def likely_match?(filename)
+    filename =~ /\.hei[cf]$/i
+  end
+
+  FormatParser.register_parser(new, natures: :image, formats: [:heif, :heic], priority: 2)
+end

data/lib/parsers/moov_parser.rb

@@ -136,7 +136,7 @@ class FormatParser::MOOVParser
 
     if stts && mdhd
       timescale = mdhd.atom_fields[:tscale]
-      sample_duration = stts.field_value(:entries).first[:sample_duration]
+      sample_duration = stts.field_value(:entries).dig(0, :sample_duration)
       if timescale.nil? || timescale == 0 || sample_duration.nil? || sample_duration == 0
         nil
       else
@@ -163,5 +163,5 @@ class FormatParser::MOOVParser
     end
   end
 
-  FormatParser.register_parser new, natures: :video, formats: FTYP_MAP.values, priority: 1
+  FormatParser.register_parser new, natures: :video, formats: FTYP_MAP.values, priority: 3
 end

data/lib/parsers/pdf_parser.rb

@@ -21,5 +21,5 @@ class FormatParser::PDFParser
     FormatParser::Document.new(format: :pdf, content_type: PDF_CONTENT_TYPE)
   end
 
-  FormatParser.register_parser new, natures: :document, formats: :pdf, priority: 1
+  FormatParser.register_parser new, natures: :document, formats: :pdf, priority: 3
 end

data/lib/parsers/webp_parser.rb

@@ -0,0 +1,162 @@
+# WebP is an image format that provides superior lossless and lossy compression for images on the web, with support for
+# transparency. It uses predictive coding to encode an image, predicting the values in a block of pixels based on the
+# values of neighbouring blocks. A WebP file consists of VP8 or VP8L data, and a container based on RIFF. There is also
+# an extended file format, VP8X, that optionally encodes various information such as the color profile, animation
+# control data, transparency, and EXIF and/or XMP metadata.
+#
+# For more information, visit https://developers.google.com/speed/webp.
+#
+# TODO: Decide how to determine color mode (depends on variant, transformations, flags, etc.; maybe not worth it).
+
+class FormatParser::WebpParser
+  include FormatParser::EXIFParser
+  include FormatParser::IOUtils
+
+  WEBP_MIME_TYPE = 'image/webp'
+
+  def likely_match?(filename)
+    filename =~ /\.webp$/i
+  end
+
+  def call(io)
+    @buf = FormatParser::IOConstraint.new(io)
+
+    # All WebP files start with the following 20 bytes:
+    #
+    # Offset  | Description
+    # -------------------------------------------------------------------------------------
+    # 0...3   | "RIFF" (Since WebP is based on the RIFF file container format).
+    # 4...7   | The size of the file in bytes - 8 bytes.
+    # 8...11  | "WEBP" (To signify that this is a WebP file).
+    # 12...15 | The VB8 variant in use ("VB8 ", "VP8L" or "VB8X")
+    # 16...19 | The length of the VB8 data in bytes (i.e. The size of the file - 20 bytes).
+    riff, webp, variant = safe_read(@buf, 20).unpack('A4x4A4A4')
+    return unless riff == 'RIFF' && webp == 'WEBP'
+    read_data(variant)
+  end
+
+  private
+
+  def read_data(variant)
+    case variant
+    when 'VP8' # Lossy
+      read_lossy_data
+    when 'VP8L' # Lossless
+      read_lossless_data
+    when 'VP8X' # Extended
+      read_extended_data
+    else
+      nil
+    end
+  end
+
+  def read_lossy_data
+    # Encoded as a single VP8 key frame - a 10-byte uncompressed chunk followed by 2+ partitions of compressed data.
+    # The first 6 bytes of this chunk contains information that is mostly relevant when using VP8 as a video
+    # compression format, and can be ignored.
+    safe_skip(@buf, 6)
+
+    # The subsequent 4 bytes contain the image width and height, respectively, as 16-bit unsigned little endian
+    # integers.
+    width, height = safe_read(@buf, 4).unpack('S<S<')
+    create_image(width, height)
+  end
+
+  def read_lossless_data
+    # There is a single byte signature, 0x2F, that we can disregard.
+    safe_skip(@buf, 1)
+
+    # The subsequent 4 bytes contain the image width and height, respectively, as 14-bit unsigned little endian
+    # integers (minus one). The 4 remaining bits consist of a 1-bit flag indicating whether alpha is used, and a 3-bit
+    # version that is always zero.
+    dimensions = read_little_endian_int_32
+    width = (dimensions & 0x3fff) + 1
+    height = (dimensions >> 14 & 0x3fff) + 1
+    has_transparency = (dimensions >> 28 & 0x1) == 1
+
+    create_image(width, height, has_transparency: has_transparency)
+  end
+
+  def read_extended_data
+    # After the common RIFF header bytes, the extended file format has a series of 1-bit flags to signify the presence
+    # of optional information. These flags are as follows:
+    #
+    # |0|1|2|3|4|5|6|7|
+    # +-+-+-+-+-+-+-+-+
+    # |Rsv|I|L|E|X|A|R|
+    #
+    # Where:
+    #   - Rsv & R = Reserved - Should be 0.
+    #   - I = Set if file contains an ICC profile.
+    #   - L = Set if file contains transparency information.
+    #   - E = Set if file contains Exif metadata.
+    #   - X = Set if file contains XMP metadata.
+    #   - A = Set if file is an animated image.
+    flags = read_int_8
+    has_transparency = flags & 0x10 != 0
+    has_exif_metadata = flags & 0x08 != 0
+    has_xmp_metadata = flags & 0x04 != 0
+    has_multiple_frames = flags & 0x02 != 0
+
+    # The flags are followed by three reserved bytes of zeros, and then by the width and height, respectively - each
+    # occupying three bytes and each one less than the actual canvas measurements.
+    safe_skip(@buf, 3)
+    dimensions = safe_read(@buf, 6).unpack('VS')
+    width = (dimensions[0] & 0xffffff) + 1
+    height = (dimensions[0] >> 24 | dimensions[1] << 8 & 0xffffff) + 1
+
+    image = create_image(width, height, has_multiple_frames: has_multiple_frames, has_transparency: has_transparency)
+    augment_image(image) if has_exif_metadata || has_xmp_metadata || has_multiple_frames
+    image
+  end
+
+  def create_image(width, height, has_multiple_frames: false, has_transparency: false)
+    FormatParser::Image.new(
+      content_type: WEBP_MIME_TYPE,
+      format: :webp,
+      has_multiple_frames: has_multiple_frames,
+      has_transparency: has_transparency,
+      height_px: height,
+      width_px: width
+    )
+  end
+
+  def augment_image(image)
+    # We're going to scan the file looking for the EXIF, XMP and/or ANMF chunks.
+    intrinsics = {}
+    num_frames = 0
+    loop do
+      # Try to read the next chunk header, and break the loop if we've reached EOF.
+      begin
+        fourcc, chunk_size = safe_read(@buf, 8).unpack('A4V')
+      rescue InvalidRead
+        break
+      end
+
+      # Padding byte of 0 added if chunk size is odd.
+      safe_skip(@buf, 1) if chunk_size.odd?
+
+      case fourcc
+      when 'EXIF'
+        exif = exif_from_tiff_io(StringIO.new(safe_read(@buf, chunk_size)))
+        # We use ||= here as one Exif chunk at most should be present, even though it is possible for there to be more.
+        intrinsics[:exif] ||= exif
+        image.height_px, image.width_px = image.width_px, image.height_px if exif&.rotated?
+        image.orientation = exif&.orientation_sym
+      when 'XMP'
+        # We use ||= here as one XMP chunk at most should be present, even though it is possible for there to be more.
+        intrinsics[:xmp] ||= safe_read(@buf, chunk_size)
+      when 'ANMF'
+        num_frames += 1 if image.has_multiple_frames
+        safe_skip(@buf, chunk_size)
+      else
+        safe_skip(@buf, chunk_size)
+      end
+    end
+
+    image.intrinsics = intrinsics unless intrinsics.empty?
+    image.num_animation_or_video_frames = num_frames if num_frames > 0
+  end
+
+  FormatParser.register_parser new, natures: [:image], formats: [:webp]
+end

data/lib/parsers/zip_parser.rb

@@ -60,5 +60,5 @@ class FormatParser::ZIPParser
     end
   end
 
-  FormatParser.register_parser new, natures: [:archive, :document], formats: :zip, priority: 2
+  FormatParser.register_parser new, natures: [:archive, :document], formats: :zip, priority: 4
 end

data/spec/parsers/heif_parser_spec.rb

@@ -0,0 +1,75 @@
+require 'spec_helper'
+
+describe FormatParser::HEIFParser do
+  it 'is able to parse single heif image with heic major brand' do
+    heif_path = fixtures_dir + 'HEIF/SingleImage.heic'
+
+    result = subject.call(File.open(heif_path, 'rb'))
+    expect(result).not_to be_nil
+    expect(result.nature).to eq(:image)
+    expect(result.format).to eq(:heic)
+    expect(result.width_px).to eq(4000)
+    expect(result.height_px).to eq(3000)
+    expect(result.content_type).to eq('image/heic')
+    expect(result.intrinsics[:compatible_brands].should =~ ['mif1', 'heic'])
+  end
+
+  it 'is able to parse single heif image with mif1 major brand' do
+    heif_path = fixtures_dir + 'HEIF/SingleImage_Autumn.heic'
+
+    result = subject.call(File.open(heif_path, 'rb'))
+    expect(result).not_to be_nil
+    expect(result.nature).to eq(:image)
+    expect(result.format).to eq(:heif)
+    expect(result.width_px).to eq(1440)
+    expect(result.height_px).to eq(960)
+    expect(result.content_type).to eq('image/heif')
+    expect(result.intrinsics[:compatible_brands].should =~ ['mif1', 'heic'])
+  end
+
+  it 'is able to parse image collection with mif1 major brand' do
+    heif_path = fixtures_dir + 'HEIF/ImageCollection.heic'
+
+    result = subject.call(File.open(heif_path, 'rb'))
+    expect(result).not_to be_nil
+    expect(result.nature).to eq(:image)
+    expect(result.format).to eq(:heif)
+    expect(result.width_px).to eq(1440)
+    expect(result.height_px).to eq(960)
+    expect(result.content_type).to eq('image/heif')
+  end
+
+  it 'is able to parse image collection with colour info' do
+    heif_path = fixtures_dir + 'HEIF/SingleImage_Autumn_WithColourInfo.heic'
+
+    result = subject.call(File.open(heif_path, 'rb'))
+    expect(result).not_to be_nil
+    expect(result.nature).to eq(:image)
+    expect(result.format).to eq(:heic)
+    expect(result.width_px).to eq(1440)
+    expect(result.height_px).to eq(960)
+    colour_info = result.intrinsics[:colour_info]
+    expect(colour_info[:colour_primaries]).to eq(28259)
+    expect(colour_info[:transfer_characteristics]).to eq(27768)
+    expect(colour_info[:matrix_coefficients]).to eq(2)
+    expect(result.content_type).to eq('image/heic')
+    expect(result.intrinsics[:compatible_brands].should =~ ['mif1', 'heic'])
+  end
+
+  it 'is able to parse image collection with pixel info' do
+    heif_path = fixtures_dir + 'HEIF/SingleImage_Autumn_WithColourInfo.heic'
+
+    result = subject.call(File.open(heif_path, 'rb'))
+    expect(result).not_to be_nil
+    expect(result.nature).to eq(:image)
+    expect(result.format).to eq(:heic)
+    expect(result.width_px).to eq(1440)
+    expect(result.height_px).to eq(960)
+    pixel_info = result.intrinsics[:pixel_info]
+    expect(pixel_info[0][:bits_in_channel_2]).to eq(8)
+    expect(pixel_info[1][:bits_in_channel_3]).to eq(8)
+    expect(pixel_info[2][:bits_in_channel_4]).to eq(8)
+    expect(result.content_type).to eq('image/heic')
+    expect(result.intrinsics[:compatible_brands].should =~ ['mif1', 'heic'])
+  end
+end

data/spec/parsers/webp_parser_spec.rb

@@ -0,0 +1,121 @@
+require 'spec_helper'
+
+describe FormatParser::WebpParser do
+  it 'does not parse files with an invalid RIFF header' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/invalid-header.webp', 'rb'))
+    expect(result).to be_nil
+  end
+
+  it 'does not parse files with an unrecognised variant' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/unrecognised-variant.webp', 'rb'))
+    expect(result).to be_nil
+  end
+
+  it 'successfully parses lossy (VP8) WebP files' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/lossy.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(false)
+    expect(result.has_transparency).to eq(false)
+    expect(result.height_px).to eq(181)
+    expect(result.intrinsics).to be_nil
+    expect(result.orientation).to be_nil
+    expect(result.width_px).to eq(65)
+  end
+
+  it 'successfully parses lossless WebP files' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/lossless.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(false)
+    expect(result.has_transparency).to eq(false)
+    expect(result.height_px).to eq(181)
+    expect(result.intrinsics).to be_nil
+    expect(result.orientation).to be_nil
+    expect(result.width_px).to eq(65)
+  end
+
+  it 'successfully parses lossless WebP files with an alpha channel' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/lossless-alpha.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(false)
+    expect(result.has_transparency).to eq(true)
+    expect(result.height_px).to eq(181)
+    expect(result.intrinsics).to be_nil
+    expect(result.orientation).to be_nil
+    expect(result.width_px).to eq(65)
+  end
+
+  it 'successfully parses extended WebP files' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/extended.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(false)
+    expect(result.has_transparency).to eq(false)
+    expect(result.height_px).to eq(181)
+    expect(result.intrinsics).to be_nil
+    expect(result.orientation).to be_nil
+    expect(result.width_px).to eq(65)
+  end
+
+  it 'successfully parses extended WebP files with an alpha channel' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/extended-alpha.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(false)
+    expect(result.has_transparency).to eq(true)
+    expect(result.height_px).to eq(181)
+    expect(result.intrinsics).to be_nil
+    expect(result.orientation).to be_nil
+    expect(result.width_px).to eq(65)
+  end
+
+  it 'successfully parses extended WebP files with Exif metadata' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/extended-exif.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(false)
+    expect(result.has_transparency).to eq(false)
+    expect(result.height_px).to eq(181)
+    expect(result.intrinsics).not_to be_nil
+    expect(result.intrinsics[:exif]).not_to be_nil
+    expect(result.intrinsics[:exif].image_length).to eq(result.height_px)
+    expect(result.intrinsics[:exif].image_width).to eq(result.width_px)
+    expect(result.orientation).to eq(:top_left)
+    expect(result.width_px).to eq(65)
+  end
+
+  it 'successfully parses extended WebP files with XMP metadata' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/extended-xmp.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(false)
+    expect(result.has_transparency).to eq(false)
+    expect(result.height_px).to eq(181)
+    expect(result.intrinsics).not_to be_nil
+    expect(result.intrinsics[:xmp]).not_to be_nil
+    expect(result.orientation).to be_nil
+    expect(result.width_px).to eq(65)
+  end
+
+  it 'successfully parses extended WebP files with animation' do
+    result = subject.call(File.open(fixtures_dir + 'WEBP/extended-animation.webp', 'rb'))
+    expect(result).not_to be_nil
+    expect(result.content_type).to eq('image/webp')
+    expect(result.format).to eq(:webp)
+    expect(result.has_multiple_frames).to eq(true)
+    expect(result.has_transparency).to eq(true)
+    expect(result.height_px).to eq(211)
+    expect(result.intrinsics).to be_nil
+    expect(result.orientation).to be_nil
+    expect(result.width_px).to eq(211)
+  end
+end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Noah Berman
 - Julik Tarkhanov
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-04-07 00:00:00.000000000 Z
+date: 2022-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -238,6 +238,7 @@ files:
 - lib/parsers/fdx_parser.rb
 - lib/parsers/flac_parser.rb
 - lib/parsers/gif_parser.rb
+- lib/parsers/heif_parser.rb
 - lib/parsers/jpeg_parser.rb
 - lib/parsers/m3u_parser.rb
 - lib/parsers/moov_parser.rb
@@ -251,6 +252,7 @@ files:
 - lib/parsers/psd_parser.rb
 - lib/parsers/tiff_parser.rb
 - lib/parsers/wav_parser.rb
+- lib/parsers/webp_parser.rb
 - lib/parsers/zip_parser.rb
 - lib/parsers/zip_parser/file_reader.rb
 - lib/parsers/zip_parser/office_formats.rb
@@ -278,6 +280,7 @@ files:
 - spec/parsers/fdx_parser_spec.rb
 - spec/parsers/flac_parser_spec.rb
 - spec/parsers/gif_parser_spec.rb
+- spec/parsers/heif_parser_spec.rb
 - spec/parsers/jpeg_parser_spec.rb
 - spec/parsers/m3u_parser_spec.rb
 - spec/parsers/moov_parser_spec.rb
@@ -289,6 +292,7 @@ files:
 - spec/parsers/psd_parser_spec.rb
 - spec/parsers/tiff_parser_spec.rb
 - spec/parsers/wav_parser_spec.rb
+- spec/parsers/webp_parser_spec.rb
 - spec/parsers/zip_parser_spec.rb
 - spec/read_limiter_spec.rb
 - spec/read_limits_config_spec.rb
@@ -300,7 +304,7 @@ licenses:
 - MIT (Hippocratic)
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -315,8 +319,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.4
-signing_key: 
+rubygems_version: 3.2.33
+signing_key:
 specification_version: 4
 summary: A library for efficient parsing of file metadata
 test_files: []