format_parser 0.5.2 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90c91543dad4ef03fec136ae3b8fd3385ee518e5bbbebbba867a75142a1780f0
-  data.tar.gz: 5e7d286eefc1170391b6ff896d20e24f39682d2e5e4dc8a7c0b4c67df82eac92
+  metadata.gz: fbc5d4476d94ccbeb0f50c3bd0406cb57d1fc5fa640afbbf8057d874ecc128a3
+  data.tar.gz: fa4421fa509d0f0528beb4c4b19dc1a63cd66fbed29142ed33b9b2aeae9067c0
 SHA512:
-  metadata.gz: 864c0a272275c3d877ecf6fac7b2d7a080876074ee8e7b5858270da1418737cca3b8abc7290649a41f662209c2cf953f714334dc1adb6dabd68187fa3e2fcd25
-  data.tar.gz: cb04914d011588f6b53976fa5123f13b6ef9f5eb10c40bae1ea4e7c13ba51f382f10721d865164a4f8460554cb28881db7c8f4ff2217d4b1d0a115dbe5fbdf5e
+  metadata.gz: 6c3abee783f5b2c35da44231bbec2bfabd1ff25cafe45716694751f865e4cae6c71286dcf2053770f925391f8b3e0cef72daa351c43de03d0ed8b0900e4eabad
+  data.tar.gz: ab9ef11e37f5d2610ded657483a688876402cde506af5f830dd2f60fe2a8d432c182227a5c514050e79cfb059906fca2d384b366a6e64969ad7f5b5de81b8f43

data/README.md

@@ -185,3 +185,6 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
 
 ### .docx
 - The .docx files were generated by the project maintainers
+
+### .key
+- The `keynote_recognized_as_jpeg.key` file was created by the project maintainers

data/lib/care.rb

@@ -4,7 +4,7 @@
 # is only available via HTTP, for example, we can have less
 # fetches and have them return more data for one fetch
 class Care
-  DEFAULT_PAGE_SIZE = 64 * 1024
+  DEFAULT_PAGE_SIZE = 128 * 1024
 
   class IOWrapper
     def initialize(io, cache = Cache.new(DEFAULT_PAGE_SIZE))

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.5.2'
+  VERSION = '0.6.0'
 end

data/lib/parsers/exif_parser.rb

@@ -1,4 +1,3 @@
-require 'exifr/jpeg'
 require 'exifr/tiff'
 require 'delegate'
 
@@ -48,22 +47,16 @@ class FormatParser::EXIFParser
     :left_bottom
   ]
 
-  def initialize(filetype, file_io)
-    @filetype = filetype
-    @file_io = IOExt.new(file_io)
+  def initialize(io_blob_with_exif_data)
+    @exif_io = IOExt.new(io_blob_with_exif_data)
     @exif_data = nil
     @orientation = nil
     @height = nil
     @width = nil
   end
 
-  def scan_image_exif
-    # Without the magic bytes EXIFR throws an error
-    @file_io.seek(0)
-    raw_exif_data = EXIFR::JPEG.new(@file_io) if @filetype == :jpeg
-    # Return if it's a CR2, which we don't parse yet
-    return if cr2_check(@file_io)
-    raw_exif_data = EXIFR::TIFF.new(@file_io) if @filetype == :tiff
+  def scan_image_tiff
+    raw_exif_data = EXIFR::TIFF.new(@exif_io)
     # For things that we don't yet have a parser for
     # we make the raw exif result available
     @exif_data = raw_exif_data
@@ -80,10 +73,4 @@ class FormatParser::EXIFParser
   def valid_orientation?(value)
     (1..ORIENTATIONS.length).include?(value)
   end
-
-  def cr2_check(_file_io)
-    @file_io.seek(8)
-    cr2_check_bytes = @file_io.read(2)
-    cr2_check_bytes == 'CR'
-  end
 end

data/lib/parsers/jpeg_parser.rb

@@ -9,6 +9,7 @@ class FormatParser::JPEGParser
   EOI_MARKER  = 0xD9  # end of image
   SOS_MARKER  = 0xDA  # start of stream
   APP1_MARKER = 0xE1  # maybe EXIF
+  EXIF_MAGIC_STRING = "Exif\0\0".b
 
   def call(io)
     @buf = FormatParser::IOConstraint.new(io)
@@ -34,29 +35,43 @@ class FormatParser::JPEGParser
     signature = read_next_marker
     return unless signature == SOI_MARKER
 
+    markers_start_at = @buf.pos
+
+    # Keynote files start with a series of _perfectly_ valid
+    # JPEG markers, probably for icon previews or QuickLook.
+    # We have to detect those and reject them earlier. We can
+    # make use of our magic ZIP reader to get there.
+    return if probably_keynote_zip?
+
+    @buf.seek(markers_start_at)
+
     while marker = read_next_marker
       case marker
       when *SOF_MARKERS
         scan_start_of_frame
       when EOI_MARKER, SOS_MARKER
+        # When we reach "End of image" or "Start of scan" markers
+        # we are transitioning into the image data that we don't need
+        # or we have reached EOF.
         break
       when APP1_MARKER
         scan_app1_frame
       else
         skip_frame
       end
+    end
 
-      # Return at the earliest possible opportunity
-      if @width && @height
-        return  FormatParser::Image.new(
-          format: :jpg,
-          width_px: @width,
-          height_px: @height,
-          orientation: @orientation,
-          intrinsics: @intrinsics,
-        )
-      end
+    # Return at the earliest possible opportunity
+    if @width && @height
+      return  FormatParser::Image.new(
+        format: :jpg,
+        width_px: @width,
+        height_px: @height,
+        orientation: @orientation,
+        intrinsics: @intrinsics,
+      )
     end
+
     nil # We could not parse anything
   rescue InvalidStructure
     nil # Due to the way JPEG is structured it is possible that some invalid inputs will get caught
@@ -86,19 +101,40 @@ class FormatParser::JPEGParser
   end
 
   def scan_app1_frame
-    frame = @buf.read(8)
-    if frame.include?('Exif')
-      scanner = FormatParser::EXIFParser.new(:jpeg, @buf)
-      if scanner.scan_image_exif
-        @exif_output = scanner.exif_data
-        @orientation = scanner.orientation unless scanner.orientation.nil?
-        @intrinsics[:exif_pixel_x_dimension] = @exif_output.pixel_x_dimension
-        @intrinsics[:exif_pixel_y_dimension] = @exif_output.pixel_y_dimension
-        @width = scanner.width
-        @height = scanner.height
-      end
+    # Read the entire EXIF frame at once to not overload the number of reads. If we don't,
+    # EXIFR parses our file from the very beginning and does the same parsing we do, just
+    # the second time around. What we care about, rather, is the EXIF data only. So we will
+    # pry it out of the APP1 frame and parse it as the TIFF segment - which is what EXIFR
+    # does under the hood.
+    app1_frame_content_length = read_short - 2
+    app1_frame_bytes = safe_read(@buf, app1_frame_content_length)
+
+    maybe_exif_magic_str = app1_frame_bytes[0..5]
+    maybe_exif_data = app1_frame_bytes[6..-1]
+    if maybe_exif_magic_str == EXIF_MAGIC_STRING
+      scanner = FormatParser::EXIFParser.new(StringIO.new(maybe_exif_data))
+      scanner.scan_image_tiff
+
+      @exif_output = scanner.exif_data
+      @orientation = scanner.orientation unless scanner.orientation.nil?
+      @intrinsics[:exif_pixel_x_dimension] = @exif_output.pixel_x_dimension
+      @intrinsics[:exif_pixel_y_dimension] = @exif_output.pixel_y_dimension
+      # Save these two for later, when we decide to provide display width /
+      # display height in addition to pixel buffer width / height. These two
+      # are _different concepts_. Imagine you have an image shot with a camera
+      # in portrait orientation, and the camera has an anamorphic lens. That
+      # anamorpohic lens is a smart lens, and therefore transmits pixel aspect
+      # ratio to the camera, and the camera encodes that aspect ratio into the
+      # image metadata. If we want to know what size our _pixel buffer_ will be,
+      # and how to _read_ the pixel data (stride/interleaving) - we need the
+      # pixel buffer dimensions. If we want to know what aspect and dimensions
+      # our file is going to have _once displayed_ and _once pixels have been
+      # brought to the right orientation_ we need to work with **display dimensions**
+      # which can be remarkably different from the pixel buffer dimensions.
+      @exif_width = scanner.width
+      @exif_height = scanner.height
     end
-  rescue EXIFR::MalformedJPEG
+  rescue EXIFR::MalformedTIFF
     # Not a JPEG or the Exif headers contain invalid data, or
     # an APP1 marker was detected in a file that is not a JPEG
   end
@@ -113,5 +149,10 @@ class FormatParser::JPEGParser
     safe_skip(@buf, length)
   end
 
+  def probably_keynote_zip?
+    reader = FormatParser::ZIPParser::FileReader.new
+    reader.zip?(@buf)
+  end
+
   FormatParser.register_parser self, natures: :image, formats: :jpg
 end

data/lib/parsers/tiff_parser.rb

@@ -13,8 +13,8 @@ class FormatParser::TIFFParser
     return if !endianness || cr2_check(io)
 
     w, h = read_tiff_by_endianness(io, endianness)
-    scanner = FormatParser::EXIFParser.new(:tiff, io)
-    scanner.scan_image_exif
+    scanner = FormatParser::EXIFParser.new(io)
+    scanner.scan_image_tiff
     FormatParser::Image.new(
       format: :tif,
       width_px: w,

data/lib/parsers/zip_parser.rb

@@ -6,8 +6,10 @@ class FormatParser::ZIPParser
   include OfficeFormats
 
   def call(io)
+    io = FormatParser::IOConstraint.new(io)
+
     reader = FileReader.new
-    entries = reader.read_zip_structure(io: FormatParser::IOConstraint.new(io))
+    entries = reader.read_zip_structure(io: io)
 
     filenames_set = Set.new
     entries_archive = entries.map do |ze|

data/lib/parsers/zip_parser/file_reader.rb

@@ -195,6 +195,25 @@ class FormatParser::ZIPParser::FileReader
     entries
   end
 
+  # Tells whether the given IO is likely to be a ZIP file without
+  # performing too many detailed reads
+  #
+  # @param io[#tell, #seek, #read, #size] an IO-ish object
+  # @return [Boolean]
+  def zip?(io)
+    zip_file_size = io.size
+    eocd_offset = get_eocd_offset(io, zip_file_size)
+    zip64_end_of_cdir_location = get_zip64_eocd_location(io, eocd_offset)
+    if zip64_end_of_cdir_location
+      num_files_and_central_directory_offset_zip64(io, zip64_end_of_cdir_location)
+    else
+      num_files_and_central_directory_offset(io, eocd_offset)
+    end
+    true
+  rescue Error
+    false
+  end
+
   private
 
   def skip_ahead_2(io)

data/spec/esoteric_formats_spec.rb

@@ -11,6 +11,6 @@ describe 'Parsing esoteric files and files causing ambiguous detection' do
   it 'does not return a result for a Keynote file when it mistakes it for a JPEG, and does not raise any errors' do
     jpeg_path = fixtures_dir + '/JPEG/keynote_recognized_as_jpeg.key'
     result = FormatParser.parse(File.open(jpeg_path, 'rb'))
-    expect(result).to be_nil
+    expect(result.nature).to eq(:archive)
   end
 end

data/spec/parsers/exif_parser_spec.rb

@@ -1,27 +1,12 @@
 require 'spec_helper'
 
 describe FormatParser::EXIFParser do
-  describe 'is able to correctly parse orientation for all the JPEG EXIF examples from FastImage' do
-    Dir.glob(fixtures_dir + '/exif-orientation-testimages/jpg/*.jpg').each do |jpeg_path|
-      filename = File.basename(jpeg_path)
-      it "is able to parse #{filename}" do
-        parser = FormatParser::EXIFParser.new(:jpeg, File.open(jpeg_path, 'rb'))
-        parser.scan_image_exif
-        expect(parser).not_to be_nil
-
-        expect(parser.orientation).to be_kind_of(Symbol)
-        # Filenames in this dir correspond with the orientation of the file
-        expect(filename.include?(parser.orientation.to_s)).to be true
-      end
-    end
-  end
-
   describe 'is able to correctly parse orientation for all the TIFF EXIF examples from FastImage' do
     Dir.glob(fixtures_dir + '/exif-orientation-testimages/tiff-*/*.tif').each do |tiff_path|
       filename = File.basename(tiff_path)
       it "is able to parse #{filename}" do
-        parser = FormatParser::EXIFParser.new(:tiff, File.open(tiff_path, 'rb'))
-        parser.scan_image_exif
+        parser = FormatParser::EXIFParser.new(File.open(tiff_path, 'rb'))
+        parser.scan_image_tiff
         expect(parser).not_to be_nil
 
         expect(parser.orientation).to be_kind_of(Symbol)

data/spec/parsers/jpeg_parser_spec.rb

@@ -49,10 +49,9 @@ describe FormatParser::JPEGParser do
     expect(result.intrinsics).to eq(exif_pixel_x_dimension: 8214, exif_pixel_y_dimension: 5476)
   end
 
-  it 'fails correctly (with the right exception being raised) on a Keynote file' do
-    jpeg_path = fixtures_dir + '/JPEG/keynote_recognized_as_jpeg.key'
-    expect {
-      subject.call(File.open(jpeg_path, 'rb'))
-    }.to raise_error(FormatParser::IOUtils::InvalidRead)
+  it 'does not return a result for a Keynote document' do
+    key_path = fixtures_dir + '/JPEG/keynote_recognized_as_jpeg.key'
+    result = subject.call(File.open(key_path, 'rb'))
+    expect(result).to be_nil
   end
 end

data/spec/parsers/zip_parser_spec.rb

@@ -99,4 +99,18 @@ describe FormatParser::ZIPParser do
     expect(first_entry.filename).to eq('Li��nia Extreme//')
     expect(first_entry.type).to eq(:directory)
   end
+
+  describe 'FileReader#zip?' do
+    it 'correctly detects all the ZIP files as such' do
+      reader = described_class::FileReader.new
+      Dir.glob(fixtures_dir + '/ZIP/*.zip').each do |path|
+        expect(reader).to be_zip(File.open(path, 'rb'))
+      end
+
+      4.times do
+        blob = Random.new.bytes(1024)
+        expect(reader).not_to be_zip(StringIO.new(blob))
+      end
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.6.0
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-11 00:00:00.000000000 Z
+date: 2018-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks