format_parser 1.4.2 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a9a94233949cc72d18b433cf1ddcba0e479e8b93aa1ff2e48bda6f6d86f667b
-  data.tar.gz: 4e357bc46207e95cad52d21b2aaa1781e9c231bab02e235be29663db9722f5d9
+  metadata.gz: f3f9d1c51523af6efea2ed6f8cf63f0f573e2b0b140c1af0435e6bb26961953c
+  data.tar.gz: 84b73895b8924a0cffa286a4adaa3c270951e4074ec09db718f1a8d482b8e14b
 SHA512:
-  metadata.gz: dcf8c8aeefc6166f3645dae461aadbcc2b36e96cb7a75162586fc009d562f6f978767ff877b27d0c192b5ca3107011a1bfdda842e730e486ced02e4191b53f59
-  data.tar.gz: fbc2caafb269f5e9c249e6ffe62ea8141477589256fab1bea5d058877d571725b1e619619ef9fcd33005d42759697ae7c7575d7a79217bdedc7e96ab02ce3c1b
+  metadata.gz: cfa0a69fd35d8d3c05fff79cca1b550af8b8c6876fece5443fc41f7967fed15fe20d494ce5678e16cb2a1cdd1b0918b8ce61ef4b731f3e47b18c736ca21cdf0c
+  data.tar.gz: 0f7747775606681981367432322bb7b63e2e3c862fcd506d0b843bfc2280ae3d8dcdf334f49a9150f27a181329cf993f9cf79036d8124e75d41ca01f26edebd6

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.5.0
+* Add support for `NEF` files.
+
 ## 1.4.2
 * Fix `MP3Parser` taking precedence when parsing `WEBP` files.
 

data/README.md

@@ -14,6 +14,7 @@ and [dimensions,](https://github.com/sstephenson/dimensions) borrowing from them
 
 * TIFF
 * CR2
+* NEF
 * PSD
 * PNG
 * MP3
@@ -161,6 +162,12 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
 ### CR2
 - CR2 examples are downloaded from http://www.rawsamples.ch/ and are Creative Common Licensed.
 
+### NEF
+- NEF examples are downloaded from http://www.rawsamples.ch/ and are Creative Common Licensed.
+
+### ERF
+- ERF examples are downloaded from http://www.rawsamples.ch/ and are Creative Common Licensed.
+
 ### FLAC
 - atc_fixture_vbr.flac is a converted version of the MP3 with the same name
 - c_11k16btipcm.flac is a converted version of the WAV with the same name

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '1.4.2'
+  VERSION = '1.5.0'
 end

data/lib/parsers/exif_parser.rb

@@ -41,6 +41,13 @@ module FormatParser::EXIFParser
   end
 
   class EXIFResult < SimpleDelegator
+    attr_reader :sub_ifds_data
+
+    def initialize(exif_raw_data, sub_ifds_data = {})
+      super(exif_raw_data)
+      @sub_ifds_data = sub_ifds_data
+    end
+
     def rotated?
       orientation.to_i > 4
     end
@@ -167,10 +174,38 @@ module FormatParser::EXIFParser
   # Squash exifr's invalid date warning since we do not use that data.
   EXIFR.logger = Logger.new(nil)
 
-  def exif_from_tiff_io(constrained_io)
+  def exif_from_tiff_io(constrained_io, should_include_sub_ifds = false)
     Measurometer.instrument('format_parser.EXIFParser.exif_from_tiff_io') do
-      raw_exif_data = EXIFR::TIFF.new(IOExt.new(constrained_io))
-      raw_exif_data ? EXIFResult.new(raw_exif_data) : nil
+      extended_io = IOExt.new(constrained_io)
+      exif_raw_data = EXIFR::TIFF.new(extended_io)
+
+      return unless exif_raw_data
+
+      sub_ifds_data = {}
+      if should_include_sub_ifds
+        sub_ifds_offsets = exif_raw_data.flat_map(&:sub_ifds).compact
+        sub_ifds_data = load_sub_ifds(extended_io, sub_ifds_offsets)
+      end
+
+      EXIFResult.new(exif_raw_data, sub_ifds_data)
+    end
+  end
+
+  private
+
+  # Reads exif data from subIFDs. This is important for NEF files.
+  def load_sub_ifds(extended_io, sub_ifds_offsets)
+    # Returning an hash of subIFDs using offsets as keys
+    # {
+    #    123 => { subIFD data...}
+    #    456 => { another subIFD data...}
+    # }
+    return {} if sub_ifds_offsets.empty?
+
+    EXIFR::TIFF::Data.open(extended_io) do |data|
+      sub_ifds_offsets.map do |sub_ifd_offset|
+        [sub_ifd_offset, EXIFR::TIFF::IFD.new(data, sub_ifd_offset)]
+      end.to_h
     end
   end
 

data/lib/parsers/nef_parser.rb

@@ -0,0 +1,69 @@
+class FormatParser::NEFParser
+  include FormatParser::IOUtils
+  include FormatParser::EXIFParser
+
+  MAGIC_LE = [0x49, 0x49, 0x2A, 0x0].pack('C4')
+  MAGIC_BE = [0x4D, 0x4D, 0x0, 0x2A].pack('C4')
+  HEADER_BYTES = [MAGIC_LE, MAGIC_BE]
+  NEF_MIME_TYPE = 'image/x-nikon-nef'
+
+  SUBFILE_TYPE_FULL_RESOLUTION = 0
+  SUBFILE_TYPE_REDUCED_RESOLUTION = 1
+
+  SHOULD_PARSE_SUB_IFDS = true
+
+  def likely_match?(filename)
+    filename =~ /\.nef$/i
+  end
+
+  def call(io)
+    io = FormatParser::IOConstraint.new(io)
+
+    return unless HEADER_BYTES.include?(safe_read(io, 4))
+
+    # Because of how NEF files organize their IFDs and subIFDs, we need to dive into the subIFDs
+    # to get the actual image dimensions instead of the preview's
+    exif_data = exif_from_tiff_io(io, SHOULD_PARSE_SUB_IFDS)
+
+    return unless valid?(exif_data)
+
+    full_resolution_data = get_full_resolution_ifd(exif_data)
+
+    w = full_resolution_data.image_width || exif_data.width || exif_data.pixel_x_dimension
+    h = full_resolution_data.image_length || exif_data.height || exif_data.pixel_y_dimension
+
+    FormatParser::Image.new(
+      format: :nef,
+      width_px: w,
+      height_px: h,
+      display_width_px: exif_data.rotated? ? h : w,
+      display_height_px: exif_data.rotated? ? w : h,
+      orientation: exif_data.orientation_sym,
+      intrinsics: { exif: exif_data },
+      content_type: NEF_MIME_TYPE,
+    )
+  rescue EXIFR::MalformedTIFF
+    nil
+  end
+
+  def valid?(exif_data)
+    # NEF files should hold subIFDs and have "NIKON" or "NIKON CORPORATION" as maker
+    has_sub_ifds_data = !exif_data&.sub_ifds_data.keys.empty?
+    has_sub_ifds_data && exif_data.make&.start_with?('NIKON')
+  end
+
+  # Investigates data from all subIFDs and find the one holding the full-resolution image
+  def get_full_resolution_ifd(exif_data)
+    # Most of the time, NEF files have 2 subIFDs:
+    # First one: Thumbnail (Reduced resolution)
+    # Second one: Full resolution
+    # While this is true in most situations, there are exceptions,
+    # so we can't rely in this order alone.
+
+    exif_data.sub_ifds_data.each do |_ifd_offset, ifd_data|
+      return ifd_data if ifd_data.new_subfile_type == SUBFILE_TYPE_FULL_RESOLUTION
+    end
+  end
+
+  FormatParser.register_parser new, natures: :image, formats: :nef, priority: 4
+end

data/spec/parsers/exif_parser_spec.rb

@@ -1,15 +1,88 @@
 require 'spec_helper'
 
 describe FormatParser::EXIFParser do
-  describe 'is able to correctly parse orientation for all the TIFF EXIF examples from FastImage' do
-    Dir.glob(fixtures_dir + '/exif-orientation-testimages/tiff-*/*.tif').each do |tiff_path|
-      filename = File.basename(tiff_path)
-      it "is able to parse #{filename}" do
-        result = described_class.exif_from_tiff_io(File.open(tiff_path, 'rb'))
-        expect(result).not_to be_nil
-        expect(result.orientation_sym).to be_kind_of(Symbol)
-        # Filenames in this dir correspond with the orientation of the file
-        expect(filename).to include(result.orientation_sym.to_s)
+  describe 'EXIFParser#exif_from_tiff_io' do
+    describe 'Orientation' do
+      describe 'is able to correctly parse orientation for all the TIFF EXIF examples from FastImage' do
+        Dir.glob(fixtures_dir + '/exif-orientation-testimages/tiff-*/*.tif').each do |tiff_path|
+          filename = File.basename(tiff_path)
+          it "is able to parse #{filename}" do
+            result = described_class.exif_from_tiff_io(File.open(tiff_path, 'rb'))
+            expect(result).not_to be_nil
+            expect(result.orientation_sym).to be_kind_of(Symbol)
+            # Filenames in this dir correspond with the orientation of the file
+            expect(filename).to include(result.orientation_sym.to_s)
+          end
+        end
+      end
+
+      it 'is able to deal with an orientation tag with a tuple value for orientation' do
+        path = fixtures_dir + '/EXIF/double_orientation.exif.bin'
+        exif_data = File.open(path, 'rb') do |f|
+          described_class.exif_from_tiff_io(f)
+        end
+        expect(exif_data.orientation).to eq(1)
+      end
+    end
+
+    describe 'SubIFDs' do
+      it 'should not retrieve subIFDs data by default' do
+        path = fixtures_dir + 'NEF/RAW_NIKON_D40_SRGB.NEF'
+
+        exif_data = File.open(path, 'rb') do |f|
+          described_class.exif_from_tiff_io(f)
+        end
+
+        expect(exif_data.sub_ifds_data).not_to be_nil
+        expect(exif_data.sub_ifds_data).to eq({})
+      end
+
+      it 'is able retrieve data from all subIFDs optionally' do
+        # Verifying:
+        # {
+        #    offset_1 => { subIFD_1 data...}
+        #    offset_2 => { subIFD_2 data...}
+        # }
+
+        path = fixtures_dir + 'NEF/RAW_NIKON_D40_SRGB.NEF'
+        should_include_sub_ifds = true
+
+        exif_data = File.open(path, 'rb') do |f|
+          described_class.exif_from_tiff_io(f, should_include_sub_ifds)
+        end
+
+        offset_1, offset_2 = exif_data.sub_ifds
+        sub_ifds_data = exif_data.sub_ifds_data
+
+        expect(sub_ifds_data).not_to be_nil
+        expect(sub_ifds_data).to have_key(offset_1)
+        expect(sub_ifds_data).to have_key(offset_2)
+      end
+
+      it 'returns EXIFR IFD instances as subIFD data' do
+        # Verifying:
+        # {
+        #    offset_1 => { new_subfile_type => 1, ...}
+        #    offset_2 => { new_subfile_type => 0, ...}
+        # }
+        # we shouldn't verify everything, since we trust to EXIFR for that.
+        # making sure we are getting each subfile type should be good enough.
+
+        path = fixtures_dir + 'NEF/RAW_NIKON_D40_SRGB.NEF'
+        should_include_sub_ifds = true
+
+        exif_data = File.open(path, 'rb') do |f|
+          described_class.exif_from_tiff_io(f, should_include_sub_ifds)
+        end
+
+        offset_1, offset_2 = exif_data.sub_ifds.sort
+        first_sub_ifd = exif_data.sub_ifds_data&.[](offset_1)
+        second_sub_ifd = exif_data.sub_ifds_data&.[](offset_2)
+
+        expect(first_sub_ifd).to be_kind_of(EXIFR::TIFF::IFD)
+        expect(second_sub_ifd).to be_kind_of(EXIFR::TIFF::IFD)
+        expect(first_sub_ifd.new_subfile_type).to eq(1)
+        expect(second_sub_ifd.new_subfile_type).to eq(0)
       end
     end
   end
@@ -29,8 +102,8 @@ describe FormatParser::EXIFParser do
     end
 
     it 'returns a Hash from #to_hash' do
-      first_fake_exif = double(orientation: 1, to_hash: {foo: 123, bar: 675})
-      second_fake_exif = double(orientation: 4, to_hash: {foo: 245})
+      first_fake_exif = double(orientation: 1, to_hash: { foo: 123, bar: 675 })
+      second_fake_exif = double(orientation: 4, to_hash: { foo: 245 })
 
       stack = FormatParser::EXIFParser::EXIFStack.new([first_fake_exif, second_fake_exif])
       stack_as_hash = stack.to_hash
@@ -42,14 +115,6 @@ describe FormatParser::EXIFParser do
     end
   end
 
-  it 'is able to deal with an orientation tag which a tuple value for orientation' do
-    path = fixtures_dir + '/EXIF/double_orientation.exif.bin'
-    exif_data = File.open(path, 'rb') do |f|
-      described_class.exif_from_tiff_io(f)
-    end
-    expect(exif_data.orientation).to eq(1)
-  end
-
   describe 'IOExt' do
     it 'supports readbyte' do
       io = FormatParser::EXIFParser::IOExt.new(StringIO.new('hello'))

data/spec/parsers/nef_parser_spec.rb

@@ -0,0 +1,131 @@
+require 'spec_helper'
+
+describe FormatParser::NEFParser do
+  describe 'Sample files from rawsamples' do
+    Dir.glob(fixtures_dir + '/NEF/*.NEF').each do |file_path|
+      it "is able to parse #{File.basename(file_path)}" do
+        parsed = subject.call(File.open(file_path, 'rb'))
+
+        expect(parsed).not_to be_nil
+        expect(parsed.nature).to eq(:image)
+        expect(parsed.format).to eq(:nef)
+
+        expect(parsed.width_px).to be_kind_of(Integer)
+        expect(parsed.height_px).to be_kind_of(Integer)
+
+        expect(parsed.display_width_px).to be_kind_of(Integer)
+        expect(parsed.display_height_px).to be_kind_of(Integer)
+
+        expect(parsed.orientation).to be_kind_of(Symbol)
+
+        expect(parsed.intrinsics[:exif]).not_to be_nil
+      end
+    end
+  end
+
+  describe 'Image Dimensions' do
+    it 'parses dimensions properly for a given fixture' do
+      # The default parser from EXIFr returns the dimensions from the embedded
+      # thumbnails as being the image's actual dimensions.
+      # We make sure we properly deal with this.
+
+      parsed = subject.call(File.open("#{fixtures_dir}/NEF/RAW_NIKON_1S2.NEF", 'rb'))
+
+      # Raw Image dimensions
+      expect(parsed.width_px).to eq(4_608)
+      expect(parsed.height_px).to eq(3_080)
+
+      expect(parsed.orientation).to eq(:right_top)
+    end
+
+    it 'correctly adjusts display dimensions for rotated images' do
+      # This image is rotated, meaning display_width and display_height
+      # should hold swapped values from width and height
+      parsed = subject.call(File.open("#{fixtures_dir}/NEF/RAW_NIKON_1S2.NEF", 'rb'))
+
+      # Raw Image dimensions
+      expect(parsed.width_px).to eq(4_608)
+      expect(parsed.height_px).to eq(3_080)
+
+      # Raw Dimensions considering orientation
+      expect(parsed.display_width_px).to eq(3_080)
+      expect(parsed.display_height_px).to eq(4_608)
+
+      expect(parsed.orientation).to eq(:right_top)
+    end
+
+    it 'does not return dimensions from embedded previews' do
+      Dir.glob(fixtures_dir + '/NEF/*.nef').each do |file_path|
+        # By default, NEF files include 160x120 sub_ifds.
+        # This dimensions cannot be considered by the parser.
+
+        parsed = subject.call(File.open(file_path, 'rb'))
+
+        expect(parsed.width_px).not_to eq(160)
+        expect(parsed.height_px).not_to eq(120)
+      end
+    end
+
+    it 'properly extracts dimensions when there are more than 2 subIFDs in the image' do
+      # this file has 3 subIFDs, and the RAW image information is actually in the one in the middle.
+      nef_path = "#{fixtures_dir}/NEF/RAW_NIKON_D800_14bit_FX_UNCOMPRESSED.NEF"
+
+      parsed = subject.call(File.open(nef_path, 'rb'))
+
+      expect(parsed).not_to be_nil
+      expect(parsed.width_px).to eq(7424)
+      expect(parsed.height_px).to eq(4924)
+      expect(parsed.orientation).to eq(:top_left)
+    end
+
+    describe 'correctly extracts dimensions from various NEF flavors of the same file' do
+      Dir.glob(fixtures_dir + '/NEF/RAW_NIKON_D800*.NEF').each do |file_path|
+        it "is able to parse #{File.basename(file_path)}" do
+          parsed = subject.call(File.open(file_path, 'rb'))
+
+          expect(parsed).not_to be_nil
+          expect(parsed.width_px).to eq(7424)
+          expect(parsed.height_px).to eq(4924)
+        end
+      end
+    end
+  end
+
+  describe 'False-positive avoidance' do
+    it 'should return nil for regular TIFF images' do
+      parsed = subject.call(File.open("#{fixtures_dir}/TIFF/Shinbutsureijoushuincho.tiff", 'rb'))
+      expect(parsed).to be_nil
+    end
+
+    it 'should return nil for regular CR2 images' do
+      parsed = subject.call(File.open("#{fixtures_dir}/CR2/RAW_CANON_40D_SRAW_V103.CR2", 'rb'))
+      expect(parsed).to be_nil
+    end
+
+    it 'should return nil for regular ERF images' do
+      # ERF files are also TIFFs with subIFDs but they don't have a matching "maker" tag
+      parsed = subject.call(File.open("#{fixtures_dir}/ERF/RAW_EPSON_RD1.ERF", 'rb'))
+      expect(parsed).to be_nil
+    end
+  end
+
+  describe 'Parser Performance' do
+    it 'extracts dimensions from a very large NEF economically' do
+      # this file has 77.3mb
+      file_path = "#{fixtures_dir}/NEF/RAW_NIKON_D800_14bit_FX_UNCOMPRESSED.NEF"
+
+      io = File.open(file_path, 'rb')
+      io_with_stats = FormatParser::ReadLimiter.new(io)
+
+      parsed = subject.call(io_with_stats)
+
+      expect(parsed).not_to be_nil
+      expect(parsed.width_px).to eq(7424)
+      expect(parsed.height_px).to eq(4924)
+
+      expect(io_with_stats.reads).to be_within(4).of(12)
+      expect(io_with_stats.seeks).to be_within(4).of(12)
+      expect(io_with_stats.bytes).to be_within(1024).of(59000)
+    end
+  end
+end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 1.4.2
+  version: 1.5.0
 platform: ruby
 authors:
 - Noah Berman
 - Julik Tarkhanov
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-07-13 00:00:00.000000000 Z
+date: 2022-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -246,6 +246,7 @@ files:
 - lib/parsers/mp3_parser.rb
 - lib/parsers/mp3_parser/id3_extraction.rb
 - lib/parsers/mpeg_parser.rb
+- lib/parsers/nef_parser.rb
 - lib/parsers/ogg_parser.rb
 - lib/parsers/pdf_parser.rb
 - lib/parsers/png_parser.rb
@@ -286,6 +287,7 @@ files:
 - spec/parsers/moov_parser_spec.rb
 - spec/parsers/mp3_parser_spec.rb
 - spec/parsers/mpeg_parser_spec.rb
+- spec/parsers/nef_parser_spec.rb
 - spec/parsers/ogg_parser_spec.rb
 - spec/parsers/pdf_parser_spec.rb
 - spec/parsers/png_parser_spec.rb
@@ -304,7 +306,7 @@ licenses:
 - MIT (Hippocratic)
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -319,8 +321,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
-signing_key:
+rubygems_version: 3.1.6
+signing_key: 
 specification_version: 4
 summary: A library for efficient parsing of file metadata
 test_files: []