format_parser 0.16.0 → 0.16.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: e49c10ea63c475b9bc61781ea8e76f42ef9e4307
-  data.tar.gz: c625105933c8654f5aac93bae09476e32917655b
+SHA256:
+  metadata.gz: 80362b6ee92ad34550681e8c0447dcb93b11071beac25d743356a9c0b7a548f9
+  data.tar.gz: c40fbcae9bd5417420ff5ea573c24f65462f3b0cf5269fc65024e6a237c98f45
 SHA512:
-  metadata.gz: cf72d2b021f6fc407b29cbc5b336fd71e022f6a40b67a79646ea58e605c33d34b44404339e7f15cf1c3f63729712a784ac75296fde54525f51a9c6276069619e
-  data.tar.gz: c68eadacfae8062e6fb1532d3e6976c349f1d3f7a950cafb7ea0bab714a125e81d3d32bccb154ed96c2118c142f984637ed17c6a53d1ee755778836ad008a7a0
+  metadata.gz: 7f20f7206574ea204393662e29251dc079ce615c593b21c0c4c35f3ba74a08fc9582e15443f23e45771ee2cb73b5b691ee6a31d37f5fb99e847d09d4c2e003cd
+  data.tar.gz: 6a23e2bb7797a5834e0b2867096240a29ea899d2301f1496c94b2e9d35d29698dc8599bef7185785f7a26fc929bff1e9dda1f5d7a96bbfc37227b1a2d42b3c4b

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.16.1
+* All EXIF: Make sure the 0 orientation does not get silently treated as orientation 8, mislabling
+  images which are not rotated as being rotated (orientation changed)
+* All EXIF: Make sure the 0 orientation (`unknown`) is correctly passed and represented
+* JPEG: Make sure multiple EXIF tags in APP1 markers get handled correctly (via overlays)
+
 ## 0.16.0
 * Add `filename_hint` keyword argument to `FormatParser.parse`. This can hint the library to apply
   the parser that will likely match for this filename first, and the other parsers later. This helps

data/README.md

@@ -170,5 +170,10 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
 ### .docx
 - The .docx files were generated by the project maintainers
 
+### JPEG examples of EXIF  orientation
+- Downloaded from Unspash (and thus freely avaliable) - https://unsplash.com/license and have then been
+  manipulated using the [https://github.com/recurser/exif-orientation-examples](exif-orientation-examples)
+  script.
+
 ### .key
 - The `keynote_recognized_as_jpeg.key` file was created by the project maintainers

data/Rakefile

@@ -2,6 +2,7 @@ require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 require 'yard'
 require 'rubocop/rake_task'
+require 'parallel_tests/tasks'
 
 YARD::Rake::YardocTask.new(:doc) do |t|
   # The dash has to be between the two to "divide" the source files and
@@ -11,4 +12,5 @@ end
 
 RuboCop::RakeTask.new
 RSpec::Core::RakeTask.new(:spec)
-task default: [:spec, :rubocop]
+
+task default: ['parallel:spec', :rubocop]

data/format_parser.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'ks', '~> 0.0'
-  spec.add_dependency 'exifr', '~> 1.0'
+  spec.add_dependency 'exifr', '~> 1', '>= 1.3.4'
   spec.add_dependency 'id3tag', '~> 0.10', '>= 0.10.1'
   spec.add_dependency 'faraday', '~> 0.13'
   spec.add_dependency 'measurometer', '~> 1'
@@ -42,4 +42,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'pry', '~> 0.11'
   spec.add_development_dependency 'yard', '~> 0.9'
   spec.add_development_dependency 'wetransfer_style', '0.5.0'
+  spec.add_development_dependency 'parallel_tests'
 end

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.16.0'
+  VERSION = '0.16.1'
 end

data/lib/parsers/cr2_parser.rb

@@ -37,7 +37,7 @@ class FormatParser::CR2Parser
       height_px: h,
       display_width_px: exif_data.rotated? ? h : w,
       display_height_px: exif_data.rotated? ? w : h,
-      orientation: exif_data.orientation,
+      orientation: exif_data.orientation_sym,
       intrinsics: {exif: exif_data},
     )
   rescue EXIFR::MalformedTIFF

data/lib/parsers/exif_parser.rb

@@ -2,30 +2,17 @@ require 'exifr/tiff'
 require 'delegate'
 
 module FormatParser::EXIFParser
-  ORIENTATIONS = [
-    :top_left,
-    :top_right,
-    :bottom_right,
-    :bottom_left,
-    :left_top,
-    :right_top,
-    :right_bottom,
-    :left_bottom
-  ]
-  ROTATED_ORIENTATIONS = ORIENTATIONS - [
-    :bottom_left,
-    :bottom_right,
-    :top_left,
-    :top_right
-  ]
-  module MethodsMethodFix
-    # Fix a little bug in EXIFR which breaks delegators
-    # https://github.com/remvee/exifr/pull/55
-    def methods(*)
-      super() # no args
-    end
-  end
-  EXIFR::TIFF.prepend(MethodsMethodFix)
+  ORIENTATIONS = {
+    0 => :unknown, # Non-rotated
+    1 => :top_left, # Non-rotated
+    2 => :top_right, # Non-rotated
+    3 => :bottom_right, # Non-rotated
+    4 => :bottom_left, # Non-rotated
+    5 => :left_top,
+    6 => :right_top,
+    7 => :right_bottom,
+    8 => :left_bottom
+  }
 
   # EXIFR kindly requests the presence of a few more methods than what our IOConstraint
   # is willing to provide, but they can be derived from the available ones
@@ -55,7 +42,7 @@ module FormatParser::EXIFParser
 
   class EXIFResult < SimpleDelegator
     def rotated?
-      ROTATED_ORIENTATIONS.include?(orientation)
+      __getobj__.orientation.to_i > 4
     end
 
     def to_json(*maybe_coder)
@@ -65,8 +52,104 @@ module FormatParser::EXIFParser
     end
 
     def orientation
-      value = __getobj__.orientation.to_i
-      ORIENTATIONS.fetch(value - 1)
+      __getobj__.orientation.to_i
+    end
+
+    def orientation_sym
+      ORIENTATIONS.fetch(orientation)
+    end
+  end
+
+  # With some formats, multiple EXIF tag frames can be included in a single file.
+  # For example, JPEGs might have multiple APP1 markers which each contain EXIF
+  # data. The EXIF data in them, however, is not necessarily "complete" - it seems
+  # most applications assume that these blocks "overwrite" each other with the properties
+  # they specify. Probably this is done for more efficient saving - instead of overwriting
+  # the EXIF data with a modified version - which would also potentially disturb any digital
+  # signing that this data might include - the applications are supposed to follow the order
+  # in which these tags appear in the file:
+  #
+  # Take a resized image for example:
+  #
+  #   APP1 {author: 'John', pixel_width: 1024}
+  #   APP1 {pixel_width: 10}
+  #
+  # That image would get a combined EXIF of:
+  #
+  #   APP1 {author: 'John', pixel_width: 10}
+  #
+  # since the frame that comes later in the file overwrites a property. From what I see
+  # exiftools do this is the way it works.
+  #
+  # This class acts as a wrapper for this "layering" of chunks of EXIF properties, and will
+  # follow the following conventions:
+  #
+  # * When merging data for JSON conversion, it will merge it top-down. It will overwrite
+  #   any specified properties. An exception is made for orientation (see below)
+  # * When retrieving a property, it will look "from the end to the beginning" of the EXIF
+  #   dataframe stack, looking for the first dataframe which has this property with a non-nil value
+  # * When retrieving orientation, it will pick the first orientation value which is not nil
+  #   but also not 0 ("unknown orientation"). Even files in our test suite contain these.
+  class EXIFStack
+    def initialize(multiple_exif_results)
+      @multiple_exif_results = Array(multiple_exif_results)
+    end
+
+    def to_json(*maybe_coder)
+      # Let EXIF tags that come later overwrite the properties from the tags
+      # that come earlier
+      overlay = @multiple_exif_results.each_with_object({}) do |one_exif_frame, h|
+        h.merge!(one_exif_frame.to_hash)
+      end
+      # Overwrite the orientation with our custom method implementation, because
+      # it does reject 0-values.
+      overlay[:orientation] = orientation
+
+      sanitized = FormatParser::AttributesJSON._sanitize_json_value(overlay)
+      sanitized.to_json(*maybe_coder)
+    end
+
+    def orientation_sym
+      ORIENTATIONS.fetch(orientation)
+    end
+
+    def rotated?
+      orientation > 4
+    end
+
+    def orientation
+      # Retrieving an orientation "through" the sequence of EXIF tags
+      # is trickier than the method_missing case, because the value
+      # of the orientation can be 0, meaning "unknown". We need to skip through
+      # those and return the _last_ non-0 orientation, or 0 otherwise
+      @multiple_exif_results.reverse_each do |exif_tag_frame|
+        orientation_value = exif_tag_frame.orientation
+        if !orientation_value.nil? && orientation_value != 0
+          return orientation_value
+        end
+      end
+      0 # If none were found - the orientation is unknown
+    end
+
+    private
+
+    def respond_to_missing?(method_name)
+      @multiple_exif_results.last.respond_to?(method_name)
+    end
+
+    def method_missing(*a)
+      return super unless @multiple_exif_results.any?
+
+      # The EXIF tags get appended to the file, so the ones coming _later_
+      # are more specific and potentially overwrite the earlier ones. Walk
+      # through the frames in reverse (starting with one that comes last)
+      # and if it contans the requisite EXIF property, return the value
+      # from that tag.
+      @multiple_exif_results.reverse_each do |exif_tag_frame|
+        value_of = exif_tag_frame.public_send(*a)
+        return value_of if value_of
+      end
+      nil
     end
   end
 

data/lib/parsers/jpeg_parser.rb

@@ -21,7 +21,7 @@ class FormatParser::JPEGParser
     @buf = FormatParser::IOConstraint.new(io)
     @width             = nil
     @height            = nil
-    @exif_data         = nil
+    @exif_data_frames  = []
     scan
   end
 
@@ -66,16 +66,24 @@ class FormatParser::JPEGParser
 
     Measurometer.add_distribution_value('format_parser.JPEGParser.bytes_read_until_capture', @buf.pos)
 
+    # A single file might contain multiple EXIF data frames. In a JPEG this would
+    # manifest as multiple APP1 markers. The way different programs handle these
+    # differs, for us it makes the most sense to simply "flatten" them top-down.
+    # So we start with the first EXIF frame, and we then allow the APP1 markers
+    # that come later in the file to override the properties they _do_ specify.
+    flat_exif = FormatParser::EXIFParser::EXIFStack.new(@exif_data_frames)
+
     # Return at the earliest possible opportunity
     if @width && @height
+      dw, dh = flat_exif.rotated? ? [@height, @width] : [@width, @height]
       result = FormatParser::Image.new(
         format: :jpg,
         width_px: @width,
         height_px: @height,
-        display_width_px: @exif_data && @exif_data.rotated? ? @height : @width,
-        display_height_px: @exif_data && @exif_data.rotated? ? @width : @height,
-        orientation: @exif_data && @exif_data.orientation,
-        intrinsics: {exif: @exif_data},
+        display_width_px: dw,
+        display_height_px: dh,
+        orientation: flat_exif.orientation_sym,
+        intrinsics: {exif: flat_exif},
       )
 
       return result
@@ -144,7 +152,7 @@ class FormatParser::JPEGParser
 
     Measurometer.add_distribution_value('format_parser.JPEGParser.bytes_sent_to_exif_parser', exif_buf.size)
 
-    @exif_data = exif_from_tiff_io(exif_buf)
+    @exif_data_frames << exif_from_tiff_io(exif_buf)
   rescue EXIFR::MalformedTIFF
     # Not a JPEG or the Exif headers contain invalid data, or
     # an APP1 marker was detected in a file that is not a JPEG

data/lib/parsers/tiff_parser.rb

@@ -31,7 +31,7 @@ class FormatParser::TIFFParser
       height_px: h,
       display_width_px: exif_data.rotated? ? h : w,
       display_height_px: exif_data.rotated? ? w : h,
-      orientation: exif_data.orientation,
+      orientation: exif_data.orientation_sym,
       intrinsics: {exif: exif_data},
     )
   rescue EXIFR::MalformedTIFF

data/spec/parsers/exif_parser_spec.rb

@@ -11,9 +11,9 @@ describe FormatParser::EXIFParser do
       it "is able to parse #{filename}" do
         result = subject.exif_from_tiff_io(File.open(tiff_path, 'rb'))
         expect(result).not_to be_nil
-        expect(result.orientation).to be_kind_of(Symbol)
+        expect(result.orientation_sym).to be_kind_of(Symbol)
         # Filenames in this dir correspond with the orientation of the file
-        expect(filename).to include(result.orientation.to_s)
+        expect(filename).to include(result.orientation_sym.to_s)
       end
     end
   end

data/spec/parsers/jpeg_parser_spec.rb

@@ -130,4 +130,31 @@ describe FormatParser::JPEGParser do
     serialized = JSON.pretty_generate(result)
     expect(serialized).to be_kind_of(String)
   end
+
+  it 'correctly recognizes various EXIF orientations' do
+    (0..4).each do |n|
+      path = fixtures_dir + "/exif-orientation-testimages/manipulated/Landscape_#{n}.jpg"
+      result = subject.call(File.open(path, 'rb'))
+      expect(result.display_width_px).to eq(1600)
+      expect(result.display_height_px).to eq(1200)
+    end
+    (5..8).each do |n|
+      path = fixtures_dir + "/exif-orientation-testimages/manipulated/Landscape_#{n}.jpg"
+      result = subject.call(File.open(path, 'rb'))
+      expect(result.display_width_px).to eq(1600)
+      expect(result.display_height_px).to eq(1200)
+    end
+    (0..4).each do |n|
+      path = fixtures_dir + "/exif-orientation-testimages/manipulated/Portrait_#{n}.jpg"
+      result = subject.call(File.open(path, 'rb'))
+      expect(result.display_width_px).to eq(1200)
+      expect(result.display_height_px).to eq(1600)
+    end
+    (5..8).each do |n|
+      path = fixtures_dir + "/exif-orientation-testimages/manipulated/Portrait_#{n}.jpg"
+      result = subject.call(File.open(path, 'rb'))
+      expect(result.display_width_px).to eq(1200)
+      expect(result.display_height_px).to eq(1600)
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.16.0
+  version: 0.16.1
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-07-08 00:00:00.000000000 Z
+date: 2019-08-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -31,14 +31,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.4
 - !ruby/object:Gem::Dependency
   name: id3tag
   requirement: !ruby/object:Gem::Requirement
@@ -171,6 +177,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.5.0
+- !ruby/object:Gem::Dependency
+  name: parallel_tests
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |-
   A Ruby library for prying open files you can convert to a previewable format, such as video, image and audio files. It includes
     a number of parser modules that try to recover metadata useful for post-processing and layout while reading the absolute
@@ -284,7 +304,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: A library for efficient parsing of file metadata