format_parser 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 893aa7a0c9ab6b3f13930caffd0c455ef341c91f
-  data.tar.gz: e63bb62ac26c49e1958ab62eb9410058084d7d44
+  metadata.gz: 205c4099f44c0080b53e210ea18cebd4484476ba
+  data.tar.gz: d8145f8f77be44dab386585679a8c7f3ce48869d
 SHA512:
-  metadata.gz: 1ba899600ad478696d35478a57838d2cc66e75a1e4884f9f993a8a5df584184973d65effde17751dd063c2c08edb65bf8a3a3fbbb33595b2cdb614ad31fe0759
-  data.tar.gz: 3476574566c7dcab2dc7113337107c6fe182860478890c31da0ff6588ccb4d7adcb068ff4dcc3c9d2e01c43d815db4240c7d50e36f4014c69834780db675747f
+  metadata.gz: ad845e5216be2e71e81205cade268d72b1be1ee2e5721a7877a5222a03a43c0f10d18abad8fc1c553165ade0665c7bba9ef5d5a79aa954e8c587ac0d027d6d89
+  data.tar.gz: 172c0d12205c778dadefe57010c5cfe6635d5398eea385048b8e41b3558a326cba4c332e37b9259abef4589d78f31c39bffb7b9c6adb416417b771cdb2b49272

data/README.md

@@ -19,6 +19,7 @@ file_info.width_px              #=> 320
 file_info.height_px             #=> 240
 file_info.orientation           #=> :top_left
 ```
+
 If nothing is detected, the result will be `nil`.
 
 ## Design rationale

data/lib/care.rb

@@ -12,10 +12,18 @@ class Care
       @pos = 0
     end
 
+    def size
+      @io.size
+    end
+
     def seek(to)
       @pos = to
     end
 
+    def pos
+      @pos
+    end
+
     def read(n_bytes)
       read = @cache.byteslice(@io, @pos, n_bytes)
       return nil unless read && !read.empty?

data/lib/format_parser.rb

@@ -5,6 +5,7 @@ module FormatParser
   require_relative 'io_utils'
   require_relative 'read_limiter'
   require_relative 'remote_io'
+  require_relative 'io_constraint'
   require_relative 'care'
 
   PARSER_MUX = Mutex.new
@@ -17,10 +18,21 @@ module FormatParser
   end
 
   def self.parse_http(url)
-    parse(RemoteIO.new(url))
+    remote_io = RemoteIO.new(url)
+    cached_io = Care::IOWrapper.new(remote_io)
+
+    # Prefetch the first page, since it is very likely to be touched
+    # by all parsers anyway. Additionally, when using RemoteIO we need
+    # to explicitly obtain the size of the resource, which is only available
+    # after having performed at least one successful GET - at least on S3
+    cached_io.read(1); cached_io.seek(0)
+
+    parse(cached_io)
   end
 
   def self.parse(io)
+    # If the cache is preconfigured do not apply an extra layer. It is going
+    # to be preconfigured when using parse_http.
     io = Care::IOWrapper.new(io) unless io.is_a?(Care::IOWrapper)
 
     # Always instantiate parsers fresh for each input, since they might

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.1.1'
+  VERSION = '0.1.2'
 end

data/lib/io_constraint.rb

@@ -0,0 +1,37 @@
+# We deliberately want to document and restrict the
+# number of methods an IO-ish object has to implement
+# to be usable with all our parsers. This subset is fairly
+# thin and well defined, and all the various IO limiters
+# and cache facilities in the library are guaranteed to
+# support those methods. This wrapper is used to guarantee
+# that the parser can only call those specific methods and
+# nothing more. Consequently, if the parser uses a gem that
+# for some reason needs additional IO methods to be available
+# this parser has to provide it's own extensions to that end.
+#
+# The rationale for including a method in this subset is as follows:
+# we include a method if other methods can be implemented on top of it.
+# For example, should some parser desire `IO#readbyte`, it can be
+# implemented in terms of a `read()`. Idem for things like `IO#eof?`,
+# `IO#rewind` and friends.
+class FormatParser::IOConstraint
+  def initialize(io)
+    @io = io
+  end
+  
+  def read(n_bytes)
+    @io.read(n_bytes)
+  end
+  
+  def seek(absolute_offset)
+    @io.seek(absolute_offset)
+  end
+  
+  def size
+    @io.size
+  end
+
+  def pos
+    @io.pos
+  end
+end

data/lib/parsers/aiff_parser.rb

@@ -19,7 +19,7 @@ class FormatParser::AIFFParser
   ]
 
   def information_from_io(io)
-    io.seek(0)
+    io = FormatParser::IOConstraint.new(io)
     form_chunk_type, chunk_size = safe_read(io, 8).unpack('a4N')
     return unless form_chunk_type == "FORM" && chunk_size > 4
 

data/lib/parsers/dpx_parser.rb

@@ -125,6 +125,7 @@ class FormatParser::DPXParser
   HEADER_SIZE = SIZEOF[DPX_INFO] # Does not include the initial 4 bytes
 
   def information_from_io(io)
+    io = FormatParser::IOConstraint.new(io)
     magic = io.read(4)
 
     return nil unless [BE_MAGIC, LE_MAGIC].include?(magic)

data/lib/parsers/exif_parser.rb

@@ -1,9 +1,24 @@
 require 'exifr/jpeg'
 require 'exifr/tiff'
+require 'delegate'
 
 class FormatParser::EXIFParser
   include FormatParser::IOUtils
 
+  # EXIFR kindly requests the presence of getbyte and readbyte
+  # IO methods, which our constrained IO subset does not provide natively
+  class IOExt < SimpleDelegator
+    def readbyte
+      if byte = read(1)
+        byte.unpack('C').first
+      else
+        nil
+      end
+    end
+
+    alias_method :getbyte, :readbyte
+  end
+
   # Squash exifr's invalid date warning since we do not use that data.
   logger = Logger.new(nil)
   EXIFR.logger = logger
@@ -21,9 +36,9 @@ class FormatParser::EXIFParser
     :left_bottom
   ]
 
-  def initialize(filetype, file_data)
+  def initialize(filetype, file_io)
     @filetype = filetype
-    @file_data = file_data
+    @file_io = IOExt.new(file_io)
     @exif_data = nil
     @orientation = nil
     @height = nil
@@ -31,11 +46,10 @@ class FormatParser::EXIFParser
   end
 
   def scan_image_exif
-
     # Without the magic bytes EXIFR throws an error
-    @file_data.seek(0)
-    raw_exif_data = EXIFR::JPEG.new(@file_data) if @filetype == :jpeg
-    raw_exif_data = EXIFR::TIFF.new(@file_data) if @filetype == :tiff
+    @file_io.seek(0)
+    raw_exif_data = EXIFR::JPEG.new(@file_io) if @filetype == :jpeg
+    raw_exif_data = EXIFR::TIFF.new(@file_io) if @filetype == :tiff
     # For things that we don't yet have a parser for
     # we make the raw exif result available
     @exif_data = raw_exif_data

data/lib/parsers/gif_parser.rb

@@ -5,6 +5,8 @@ class FormatParser::GIFParser
   include FormatParser::IOUtils
 
   def information_from_io(io)
+    io = FormatParser::IOConstraint.new(io)
+
     header = safe_read(io, 6)
     return unless HEADERS.include?(header)
 

data/lib/parsers/jpeg_parser.rb

@@ -11,7 +11,7 @@ class FormatParser::JPEGParser
   APP1_MARKER = 0xE1  # maybe EXIF
 
   def information_from_io(io)
-    @buf = io
+    @buf = FormatParser::IOConstraint.new(io)
     @width             = nil
     @height            = nil
     @orientation       = nil

data/lib/parsers/png_parser.rb

@@ -19,8 +19,9 @@ class FormatParser::PNGParser
     safe_read(io, 8).unpack("Na4")
   end
 
-
   def information_from_io(io)
+    io = FormatParser::IOConstraint.new(io)
+
     magic_bytes = safe_read(io, PNG_HEADER_BYTES.bytesize)
     return unless magic_bytes == PNG_HEADER_BYTES
 

data/lib/parsers/psd_parser.rb

@@ -3,6 +3,8 @@ class FormatParser::PSDParser
   include FormatParser::IOUtils
 
   def information_from_io(io)
+    io = FormatParser::IOConstraint.new(io)
+
     magic_bytes = safe_read(io, 4).unpack("C4")
 
     return unless magic_bytes == PSD_HEADER

data/lib/parsers/tiff_parser.rb

@@ -7,6 +7,8 @@ class FormatParser::TIFFParser
   include FormatParser::IOUtils
 
   def information_from_io(io)
+    io = FormatParser::IOConstraint.new(io)
+
     magic_bytes = safe_read(io, 4).unpack("C4")
     endianness = scan_tiff_endianness(magic_bytes)
     return unless endianness

data/lib/read_limiter.rb

@@ -14,6 +14,14 @@ class FormatParser::ReadLimiter
     @bytes = 0
   end
 
+  def size
+    @io.size
+  end
+
+  def pos
+    @io.pos
+  end
+
   def seek(to_offset)
     @seeks += 1
     if @max_seeks && @seeks > @max_seeks

data/lib/remote_io.rb

@@ -24,11 +24,16 @@ class FormatParser::RemoteIO
     0 # always return 0
   end
 
+  # Emulates IO#pos
+  def pos
+    @pos
+  end
+
   # Emulates IO#size.
   #
-  # @return [Fixnum] the size of the remote resource
+  # @return [Integer] the size of the remote resource
   def size
-    raise "Remote size not yet obtained, need to perform at least one read() to get it" unless @remote_size
+    raise "Remote size not yet obtained, need to perform at least one read() to retrieve it" unless @remote_size
     @remote_size
   end
 
@@ -42,9 +47,14 @@ class FormatParser::RemoteIO
   # @return [String] the read bytes
   def read(n_bytes)
     http_range = (@pos..(@pos + n_bytes - 1))
-    @remote_size, body = request_range(http_range)
-    body.force_encoding(Encoding::BINARY) if body
-    body
+    maybe_size, maybe_body = request_range(http_range)
+    if maybe_size && maybe_body
+      @remote_size = maybe_size
+      @pos += maybe_body.bytesize
+      maybe_body.force_encoding(Encoding::ASCII_8BIT)
+    else
+      nil
+    end
   end
 
   protected
@@ -76,10 +86,11 @@ class FormatParser::RemoteIO
       # to be 206
       return [size, response.body]
     when 416
-      # We return `nil` as the body if we tried to read past the end of the IO,
+      # We return `nil` if we tried to read past the end of the IO,
       # which satisfies the Ruby IO convention. The caller should deal with `nil` being the result of a read()
-      # S3 will also handily _not_ supply us with the Content-Range of the actual resource
-      return [nil, nil]
+      # S3 will also handily _not_ supply us with the Content-Range of the actual resource, so we
+      # cannot hint size with this response - at lease not when working with S3
+      return nil
     when 500..599
       raise IntermittentFailure, "Server at #{@uri} replied with a #{response.status} and we might want to retry"
     else

data/spec/care_spec.rb

@@ -46,6 +46,8 @@ describe Care do
   end
 
   describe Care::IOWrapper do
+    it_behaves_like 'an IO object compatible with IOConstraint'
+
     it 'forwards calls to read() to the Care and adjusts internal offsets' do
       fake_cache_class = Class.new do
         attr_reader :recorded_calls
@@ -73,5 +75,18 @@ describe Care do
       expect(second).to eq([io_double, 2, 3])
       expect(third).to eq([io_double,  11, 5])
     end
+
+    it 'implements the complete subset of IOConstraint' do
+      methods_not_covered = Set.new(FormatParser::IOConstraint.public_instance_methods) - Set.new(Care::IOWrapper.public_instance_methods)
+      expect(methods_not_covered).to be_empty
+    end
+    
+    it 'forwards calls to size() to the underlying IO' do
+      io_double = double('IO')
+      expect(io_double).to receive(:size).and_return(123)
+
+      subject = Care::IOWrapper.new(io_double)
+      expect(subject.size).to eq(123)
+    end
   end
 end

data/spec/read_limiter_spec.rb

@@ -1,14 +1,23 @@
 require 'spec_helper'
 
-describe "ReadLimiter" do
+describe FormatParser::ReadLimiter do
   let(:io) { StringIO.new(Random.new.bytes(1024)) }
 
+  it_behaves_like 'an IO object compatible with IOConstraint'
+
   it 'does not enforce any limits with default arguments' do
     reader = FormatParser::ReadLimiter.new(io)
     2048.times { reader.seek(1) }
     2048.times { reader.read(4) }
   end
 
+  it 'passes #pos to the delegate' do
+    reader = FormatParser::ReadLimiter.new(io)
+    expect(reader.pos).to eq(0)
+    io.read(2)
+    expect(reader.pos).to eq(2)
+  end
+
   it 'enforces the number of seeks' do
     reader = FormatParser::ReadLimiter.new(io, max_seeks: 4)
     4.times { reader.seek(1) }
@@ -32,4 +41,5 @@ describe "ReadLimiter" do
       reader.read(1)
     }.to raise_error(/bytes budget \(512\) exceeded/)
   end
+
 end

data/spec/remote_fetching_spec.rb

@@ -25,6 +25,37 @@ describe 'Fetching data from HTTP remotes' do
     expect(file_information.file_nature).to eq(:image)
   end
 
+  it 'parses the JPEGs exif data' do
+    file_information = FormatParser.parse_http('http://localhost:9399/exif-orientation-testimages/jpg/top_left.jpg')
+    expect(file_information).not_to be_nil
+    expect(file_information.file_nature).to eq(:image)
+    expect(file_information.file_type).to eq(:jpg)
+    expect(file_information.orientation).to eq(:top_left)
+  end
+
+  it 'parses the TIFFs exif data' do
+    file_information = FormatParser.parse_http('http://localhost:9399/TIFF/test.tif')
+    expect(file_information).not_to be_nil
+    expect(file_information.file_nature).to eq(:image)
+    expect(file_information.file_type).to eq(:tif)
+    expect(file_information.orientation).to eq(:top_left)
+  end
+
+  describe 'is able to correctly parse orientation for all remote JPEG EXIF examples from FastImage' do
+    Dir.glob(fixtures_dir + '/exif-orientation-testimages/jpg/*.jpg').each do |jpeg_path|
+      filename = File.basename(jpeg_path)
+      it "is able to parse #{filename}" do
+        remote_jpeg_path = jpeg_path.gsub(fixtures_dir, "http://localhost:9399")
+        file_information = FormatParser.parse_http(remote_jpeg_path)
+        expect(file_information).not_to be_nil
+
+        expect(file_information.orientation).to be_kind_of(Symbol)
+        # Filenames in this dir correspond with the orientation of the file
+        expect(filename.include?(file_information.orientation.to_s)).to be true 
+      end
+    end
+  end
+  
   after(:all) do
     @server.stop
     @server_thread.join(0.5)

data/spec/remote_io_spec.rb

@@ -2,6 +2,8 @@ require 'spec_helper'
 
 describe FormatParser::RemoteIO do
 
+  it_behaves_like 'an IO object compatible with IOConstraint'
+
   it 'returns the partial content when the server supplies a 206 status' do
     rio = described_class.new("https://images.invalid/img.jpg")
     
@@ -44,6 +46,24 @@ describe FormatParser::RemoteIO do
     expect(rio.read(100)).to be_nil
   end
 
+  it 'does not overwrite size when the range cannot be satisfied and the response is 416' do
+    rio = described_class.new("https://images.invalid/img.jpg")
+
+    fake_resp = double(headers: {'Content-Range' => 'bytes 0-0/13'}, status: 206, body: 'a')
+    expect(Faraday).to receive(:get).with("https://images.invalid/img.jpg", nil, range: "bytes=0-0").and_return(fake_resp)
+    rio.read(1)
+
+    expect(rio.size).to eq(13)
+
+    fake_resp = double(headers: {}, status: 416, body: 'You jumped off the end of the file maam')
+    expect(Faraday).to receive(:get).with("https://images.invalid/img.jpg", nil, range: "bytes=100-199").and_return(fake_resp)
+
+    rio.seek(100)
+    expect(rio.read(100)).to be_nil
+
+    expect(rio.size).to eq(13)
+  end
+
   it 'raises a specific error for all 5xx responses' do
     rio = described_class.new("https://images.invalid/img.jpg")
     
@@ -53,4 +73,16 @@ describe FormatParser::RemoteIO do
     rio.seek(100)
     expect { rio.read(100) }.to raise_error(/replied with a 502 and we might want to retry/)
   end
+
+  it 'maintains and exposes #pos' do
+    rio = described_class.new("https://images.invalid/img.jpg")
+
+    expect(rio.pos).to eq(0)
+
+    fake_resp = double(headers: {'Content-Range' => 'bytes 0-0/13'}, status: 206, body: 'a')
+    expect(Faraday).to receive(:get).with("https://images.invalid/img.jpg", nil, range: "bytes=0-0").and_return(fake_resp)
+    rio.read(1)
+
+    expect(rio.pos).to eq(1)
+  end
 end

data/spec/spec_helper.rb

@@ -20,3 +20,12 @@ RSpec.configure do |c|
   c.include SpecHelpers
   c.extend SpecHelpers # makes fixtures_dir available for example groups too
 end
+
+RSpec.shared_examples "an IO object compatible with IOConstraint" do
+  it 'responds to the same subset of public instance methods' do
+    requisite_methods = FormatParser::IOConstraint.public_instance_methods - Object.public_instance_methods
+    requisite_methods.each do |requisite|
+      expect(described_class.public_instance_methods).to include(requisite), "#{described_class} must respond to #{requisite}"
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-03 00:00:00.000000000 Z
+date: 2018-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: exifr
@@ -134,6 +134,7 @@ files:
 - lib/file_information.rb
 - lib/format_parser.rb
 - lib/format_parser/version.rb
+- lib/io_constraint.rb
 - lib/io_utils.rb
 - lib/parsers/aiff_parser.rb
 - lib/parsers/dpx_parser.rb