format_parser 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 56f1a49a27872cad9d5e219919196522d322c72aafa56f610301e939e80bc8ad
-  data.tar.gz: 0a727444f42a4e37f077d95ea65a2e6ff7500057c96afd159c16eb6884c00611
+  metadata.gz: 00a7541231e6ab1309b8fc318595725c4d0ec6de6a5732973b8ed0ac5e3b6393
+  data.tar.gz: fbd9d67e3cec87e474519bdade15a700694118f75f3ede8757fcb0b0657423f5
 SHA512:
-  metadata.gz: 4f17f2af50eab4fe58ca75d2be97d71236c86e48d2804c66d45d4ee0907055baf1a37ef56c476d1ddc2b4327d41a2a2980c2f0d6fc27c54cf108185771377271
-  data.tar.gz: d91545966335fc6e5e08b2d13de204e23caae858d38e246a6d3948521cfc57a6eca5b079587a43cbf6a413d2c6aa83a6428a7e42f9a9e490cecc9f8476697687
+  metadata.gz: 05a601a35d26db59f1df83f21fbe96fceff0e96c96ec9f958aa94326a34e95c4c4c585a36182b7c392eb205f5a433758eb1f45df2d726bb2402a89015ac26279
+  data.tar.gz: e9c935e201680e6740ef764b4836ecb3adddfb7559aab5e78f8a728c1e52bce0e784862a2cf5b73b05c8a3e23c1efcac8d402304dada21d4a404011d403fb604

data/CHANGELOG.md

@@ -1,5 +1,9 @@
+## 2.2.0
+* Add support for `CR3` files.
+* Add ISO base file format decoding functionality.
+
 ## 2.1.0
-* Require minumum 2.6 ruby version.
+* Require minimum 2.6 ruby version.
 * Bring back 2.6 to test matrix, we have jruby there which is still compatible with 2.6
 * Drop `ks` dependency.
 

data/README.md

@@ -12,29 +12,37 @@ and [dimensions,](https://github.com/sstephenson/dimensions) borrowing from them
 
 ## Currently supported filetypes:
 
-* TIFF
+* AAC
+* AIFF
+* ARW
+* BMP
 * CR2
-* NEF
-* PSD
-* PNG
-* MP3
-* JPEG
-* GIF
-* PDF
+* CR3
+* DOCX
 * DPX
-* AIFF
-* WAV
-* FLAC
 * FDX
+* FLAC
+* GIF
+* HEIC
+* HEIF
+* JPEG
+* M3U
+* M4A
 * MOV
+* MP3
 * MP4
-* M4A
-* ZIP
-* DOCX, PPTX, XLSX
+* MPEG
+* NEF
 * OGG
-* MPEG, MPG
-* M3U
+* PDF
+* PNG
+* PPTX
+* PSD
+* TIFF
+* WAV
 * WEBP
+* XLSX
+* ZIP
 
 ...with [more](https://github.com/WeTransfer/format_parser/issues?q=is%3Aissue+is%3Aopen+label%3Aformats) on the way!
 
@@ -132,23 +140,6 @@ Krists Ozols for [id3tag](https://github.com/krists/id3tag) that we are using fo
 
 Unless specified otherwise in this section the fixture files are MIT licensed and from the FastImage and Dimensions projects.
 
-### JPEG
-- `divergent_pixel_dimensions_exif.jpg` is used with permission from LiveKom GmbH
-- `extended_reads.jpg` has kindly been made available by Raphaelle Pellerin for use exclusively with format_parser
-- `too_many_APP1_markers_surrogate.jpg` was created by the project maintainers
-* `orient_6.jpg` is used with permission from [Renaud Chaput](https://github.com/renchap)
-
-### AIFF
-- fixture.aiff was created by one of the project maintainers and is MIT licensed
-
-### WAV
-- c_11k16bitpcm.wav and c_8kmp316.wav are from [Wikipedia WAV](https://en.wikipedia.org/wiki/WAV#Comparison_of_coding_schemes), retrieved January 7, 2018
-- c_39064__alienbomb__atmo-truck.wav is from [freesound](https://freesound.org/people/alienbomb/sounds/39064/) and is CC0 licensed
-- c_M1F1-Alaw-AFsp.wav and d_6_Channel_ID.wav are from a [McGill Engineering site](http://www-mmsp.ece.mcgill.ca/Documents/AudioFormats/WAVE/Samples.html)
-
-### MP3
-- Cassy.mp3 has been produced by WeTransfer and may be used with the library for the purposes of testing
-
 ### AAC
 - Originals music files: “Furious Freak” and “Galway”, Kevin MacLeod (incompetech.com), Licensed under Creative Commons: By Attribution 3.0, http://creativecommons.org/licenses/by/3.0/
 - The AAC samples were converted from 'wav' format and made available [here](https://espressif-docs.readthedocs-hosted.com/projects/esp-adf/en/latest/design-guide/audio-samples.html) by Espressif Systems, as part of their audio development framework (under the ESPRESSIF MIT License).
@@ -157,69 +148,90 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
   - ff-16b-1c-44100hz.aac
   - gs-16b-2c-44100hz.aac
   - gs-16b-1c-44100hz.aac
-### FDX
-- fixture.fdx was created by one of the project maintainers and is MIT licensed
 
-### DPX
-- DPX files were created by one of the project maintainers and may be used with the library for the purposes of testing
+### AIFF
+- fixture.aiff was created by one of the project maintainers and is MIT licensed
 
-### MOOV
-- bmff.mp4 is borrowed from the [bmff](https://github.com/zuku/bmff) project
-- Test_Circular MOV files were created by one of the project maintainers and are MIT licensed
+### ARW
+- ARW examples are downloaded from http://www.rawsamples.ch/ and are Creative Common Licensed.
 
 ### CR2
 - CR2 examples are downloaded from http://www.rawsamples.ch/ and are Creative Common Licensed.
 
-### NEF
-- NEF examples are downloaded from http://www.rawsamples.ch/ and are Creative Common Licensed.
+### CR3
+- CR3 examples are downloaded from https://raw.pixls.us/ and are in the public domain (CC0).
+
+### DOCX
+- The .docx files were generated by the project maintainers
+
+### DPX
+- DPX files were created by one of the project maintainers and may be used with the library for the purposes of testing
 
 ### ERF
 - ERF examples are downloaded from http://www.rawsamples.ch/ and are Creative Common Licensed.
 
+### FDX
+- fixture.fdx was created by one of the project maintainers and is MIT licensed
+
 ### FLAC
 - atc_fixture_vbr.flac is a converted version of the MP3 with the same name
 - c_11k16btipcm.flac is a converted version of the WAV with the same name
 
-### OGG
-- `hi.ogg`,  `vorbis.ogg`, `with_confusing_magic_string.ogg`, `with_garbage_at_the_end.ogg` have been generated by the project contributors
+### JPEG
+- `divergent_pixel_dimensions_exif.jpg` is used with permission from LiveKom GmbH
+- `extended_reads.jpg` has kindly been made available by Raphaelle Pellerin for use exclusively with format_parser
+- `too_many_APP1_markers_surrogate.jpg` was created by the project maintainers
+* `orient_6.jpg` is used with permission from [Renaud Chaput](https://github.com/renchap)
 
-### M4A
-- fixture.m4a was created by one of the project maintainers and is MIT licensed
+### JPEG (EXIF orientation)
+- Downloaded from Unspash (and thus freely avaliable) - https://unsplash.com/license and have then been
+  manipulated using the [https://github.com/recurser/exif-orientation-examples](exif-orientation-examples)
+  script.
 
-### PNG
-- `simulator_screenie.png` provided by [Rens Verhoeven](https://github.com/renssies)
+### KEY
+- The `keynote_recognized_as_jpeg.key` file was created by the project maintainers
 
-### TIFF
-- `Shinbutsureijoushuincho.tiff` is obtained from Wikimedia Commons and is Creative Commons licensed
-- `IMG_9266_*.tif` and all it's variations were created by the project maintainers
+### M3U
+- The M3U fixture files were created by one of the project maintainers
 
-### ARW
-- ARW examples are downloaded from http://www.rawsamples.ch/ and are Creative Common Licensed.
+### M4A
+- fixture.m4a was created by one of the project maintainers and is MIT licensed
 
-### ZIP
-- The .zip fixture files have been created by the project maintainers
+### MOOV
+- bmff.mp4 is borrowed from the [bmff](https://github.com/zuku/bmff) project
+- Test_Circular MOV files were created by one of the project maintainers and are MIT licensed
 
-### .docx
-- The .docx files were generated by the project maintainers
+### MP3
+- Cassy.mp3 has been produced by WeTransfer and may be used with the library for the purposes of testing
 
-### .mpg and .mpeg
+### MPEG
 - The files (video 1 to 4) were downloaded from https://standaloneinstaller.com/blog/big-list-of-sample-videos-for-testers-124.html.
 - Video 5 was downloaded from https://archive.org/details/ligouHDR-HC1_sample1.
 
-### JPEG examples of EXIF  orientation
-- Downloaded from Unspash (and thus freely avaliable) - https://unsplash.com/license and have then been
-  manipulated using the [https://github.com/recurser/exif-orientation-examples](exif-orientation-examples)
-  script.
+### NEF
+- NEF examples are downloaded from http://www.rawsamples.ch/ and are Creative Common Licensed.
 
-### M3U
-- The M3U fixture files were created by one of the project maintainers
+### OGG
+- `hi.ogg`,  `vorbis.ogg`, `with_confusing_magic_string.ogg`, `with_garbage_at_the_end.ogg` have been generated by the project contributors
+
+### PNG
+- `simulator_screenie.png` provided by [Rens Verhoeven](https://github.com/renssies)
+
+### TIFF
+- `Shinbutsureijoushuincho.tiff` is obtained from Wikimedia Commons and is Creative Commons licensed
+- `IMG_9266_*.tif` and all it's variations were created by the project maintainers
+
+### WAV
+- c_11k16bitpcm.wav and c_8kmp316.wav are from [Wikipedia WAV](https://en.wikipedia.org/wiki/WAV#Comparison_of_coding_schemes), retrieved January 7, 2018
+- c_39064__alienbomb__atmo-truck.wav is from [freesound](https://freesound.org/people/alienbomb/sounds/39064/) and is CC0 licensed
+- c_M1F1-Alaw-AFsp.wav and d_6_Channel_ID.wav are from a [McGill Engineering site](http://www-mmsp.ece.mcgill.ca/Documents/AudioFormats/WAVE/Samples.html)
 
 ### WEBP
-- With the exception of extended-animation.webp, which was obtained from Wikimedia Commons and is Creative Commons 
-licensed, all of the WebP fixture files have been created by one of the project maintainers.
+- With the exception of extended-animation.webp, which was obtained from Wikimedia Commons and is Creative Commons
+  licensed, all of the WebP fixture files have been created by one of the project maintainers.
 
-### .key
-- The `keynote_recognized_as_jpeg.key` file was created by the project maintainers
+### ZIP
+- The .zip fixture files have been created by the project maintainers
 
 ## Copyright
 

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '2.1.0'
+  VERSION = '2.2.0'
 end

data/lib/io_utils.rb

@@ -27,29 +27,53 @@ module FormatParser::IOUtils
   end
 
   def read_int_8
-    safe_read(@buf, 1).unpack('C').first
+    read_bytes(1).unpack('C').first
   end
 
   def read_int_16
-    safe_read(@buf, 2).unpack('n').first
+    read_bytes(2).unpack('n').first
   end
 
   def read_int_32
-    safe_read(@buf, 4).unpack('N').first
+    read_bytes(4).unpack('N').first
+  end
+
+  def read_int_64
+    read_bytes(8).unpack('Q>').first
   end
 
   def read_little_endian_int_16
-    safe_read(@buf, 2).unpack('v').first
+    read_bytes(2).unpack('v').first
   end
 
   def read_little_endian_int_32
-    safe_read(@buf, 4).unpack('V').first
+    read_bytes(4).unpack('V').first
+  end
+
+  def read_fixed_point_16
+    read_bytes(2).unpack('C2')
+  end
+
+  def read_fixed_point_32
+    read_bytes(4).unpack('n2')
+  end
+
+  def read_fixed_point_32_2_30
+    n = read_int_32
+    [n >> 30, n & 0x3fffffff]
   end
 
   # 'n' is the number of bytes to read
-  def read_string(n)
+  def read_bytes(n)
     safe_read(@buf, n)
   end
 
+  alias_method :read_string, :read_bytes
+
+  def skip_bytes(n)
+    safe_skip(@buf, n)
+    yield if block_given?
+  end
+
   ### TODO: Some kind of built-in offset for the read
 end

data/lib/parsers/cr3_parser/decoder.rb

@@ -0,0 +1,35 @@
+require 'parsers/iso_base_media_file_format/decoder'
+
+class FormatParser::CR3Parser::Decoder < FormatParser::ISOBaseMediaFileFormat::Decoder
+  include FormatParser::EXIFParser
+
+  protected
+
+  ATOM_PARSERS = ATOM_PARSERS.merge({
+    'CMT1' => :cmt1
+  })
+  CANON_METADATA_CONTAINER_UUID = '85c0b687820f11e08111f4ce462b6a48'
+
+  def cmt1(size)
+    exif = exif_from_tiff_io(StringIO.new(read_bytes(size)))
+    if exif
+      fields = exif.to_hash
+      fields[:rotated] = exif.rotated?
+      fields[:orientation_sym] = exif.orientation_sym
+      [fields, nil]
+    else
+      [nil, nil]
+    end
+  end
+
+  def uuid(size)
+    usertype = read_bytes(16).unpack('H*').first
+    fields = { usertype: usertype }
+    children = if usertype == CANON_METADATA_CONTAINER_UUID
+      build_atom_tree(size - 16)
+    else
+      skip_bytes(size - 16)
+    end
+    [fields, children]
+  end
+end

data/lib/parsers/cr3_parser.rb

@@ -0,0 +1,51 @@
+class FormatParser::CR3Parser
+  include FormatParser::IOUtils
+  require_relative 'cr3_parser/decoder'
+
+  CR3_MIME_TYPE = 'image/x-canon-cr3'
+  MAGIC_BYTES = 'ftypcrx '
+
+  def likely_match?(filename)
+    filename =~ /\.cr3$/i
+  end
+
+  def call(io)
+    @buf = FormatParser::IOConstraint.new(io)
+
+    return unless matches_cr3_definition?
+
+    atom_tree = Decoder.new.build_atom_tree(0xffffffff, @buf)
+    moov_atom = atom_tree.find { |atom| atom.type == 'moov' }
+    cmt1_atom = moov_atom&.find_first_descendent(['CMT1'])
+    return unless cmt1_atom
+
+    width = cmt1_atom.fields[:image_width]
+    height = cmt1_atom.fields[:image_length]
+    rotated = cmt1_atom.fields[:rotated]
+    orientation = cmt1_atom.fields[:orientation_sym]
+    FormatParser::Image.new(
+      format: :cr3,
+      content_type: CR3_MIME_TYPE,
+      width_px: width,
+      height_px: height,
+      orientation: orientation,
+      display_width_px: rotated ? height : width,
+      display_height_px: rotated ? width : height,
+      intrinsics: {
+        atom_tree: atom_tree,
+        exif: cmt1_atom.fields,
+      },
+    )
+  end
+
+  private
+
+  def matches_cr3_definition?
+    skip_bytes(4)
+    matches = read_string(8) == MAGIC_BYTES
+    @buf.seek(0)
+    matches
+  end
+
+  FormatParser.register_parser new, natures: [:image], formats: [:cr3]
+end

data/lib/parsers/heif_parser.rb

@@ -179,7 +179,7 @@ class FormatParser::HEIFParser
 
   def read_item_info_box
     version = read_int_8
-    safe_skip(@buf, 3) # 0 flags
+    skip_bytes(3) # 0 flags
     entry_count = if version == 0
       read_int_16
     else
@@ -191,7 +191,7 @@ class FormatParser::HEIFParser
       return unless read_string(4) == ITEM_INFO_ENTRY
       item_info_end_pos = @buf.pos + item_info_entry_length - HEADER_LENGTH
       version = read_int_8
-      safe_skip(@buf, 3) # 0 flags
+      skip_bytes(3) # 0 flags
       case version
       when 2
         item_id = read_int_16
@@ -200,7 +200,7 @@ class FormatParser::HEIFParser
       else
         return # wrong version according to standards, hence return
       end
-      safe_skip(@buf, 2) # not interested in the item_protection_index
+      skip_bytes(2) # not interested in the item_protection_index
       item_type = read_string(4)
       content_encoding = ''
       if item_type == MIME_MARKER
@@ -212,13 +212,13 @@ class FormatParser::HEIFParser
   end
 
   def read_nil_version_and_flag
-    safe_skip(@buf, 1) # version, always 0 in this current box
-    safe_skip(@buf, 3) # flags, always 0 in this current box
+    skip_bytes(1) # version, always 0 in this current box
+    skip_bytes(3) # flags, always 0 in this current box
   end
 
   def read_primary_item_box
     version = read_int_8
-    safe_read(@buf, 3) # flags, always 0 in this current box
+    skip_bytes(3) # flags, always 0 in this current box
     @primary_item_id = if version == 0
       read_int_16
     else
@@ -332,7 +332,7 @@ class FormatParser::HEIFParser
 
   def read_item_properties_association_box
     version = read_int_8
-    safe_read(@buf, 2) # we skip the first 2 bytes of the flags (total of 3 bytes) cause we care only about the least significant bit
+    skip_bytes(2) # we skip the first 2 bytes of the flags (total of 3 bytes) cause we care only about the least significant bit
     flags = read_int_8
     entry_count = read_int_32
     item_id = 0