format_parser 1.6.0 → 2.0.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 54b56b24c97b2532bc5d7f8521aa38714111a05f2bedd5e15b7391e1005d9795
-  data.tar.gz: e33a026ab2c611a86d6ba7e35fd413455f7b99c423651fa09dcebf08ad543e0a
+  metadata.gz: d7c965b7783ecaea4802f7e585861b4400b2210fee4cb90388757530880fa074
+  data.tar.gz: fc8b7cc3f00825fa054c948a7ae817b1eee6457ffaec9e5a6b5bdd9a0b92d126
 SHA512:
-  metadata.gz: e27ec936c4b43cc6c82f896846ef3e8044a0639b18c1c2accf7bbdeffd0fe73d9c03fb2f2e9b4a477ef1f98a1043047042b8521f186e3406ca1c482be9a66abd
-  data.tar.gz: a7927cfb5fbf0a41980465186809fb830b322e83db56bab15b5a2ad897031f7269eeae85d7e61f489a5cc2e6737348309e4f93c5dee27238d06a3ed0979f0d43
+  metadata.gz: 73f774ebe540dfd54e87f89cedecfc0fabf4a97f4e2ef72afcd94edc5e0fbc344c7c67b365942e3bb915dfe76f94f038072671c259c2d366a69d64a73cbde960
+  data.tar.gz: bc1405329d521487ec4d0738c258fb12c3acdb37b6b8ecebf7451a866d5f1072cfc23774e2ecc3d7d297095ff280320756fb4cd9000de3eac447a105cf87028b

data/.github/workflows/main.yml

@@ -14,8 +14,8 @@ jobs:
       matrix:
         ruby:
           - 2.7
-          - 2.6
-          - 2.5
+          - 3.0
+          - 3.1
           - jruby
     steps:
       - name: Checkout
@@ -60,15 +60,10 @@ jobs:
       matrix:
         ruby:
           - 2.7
-          - 2.6
-          - 2.5
+          - 3.0
+          - 3.1
           - jruby
         experimental: [false]
-        include:
-          - ruby: 3.1
-            experimental: true
-          - ruby: 3.0
-            experimental: true
     steps:
       - name: Checkout
         uses: actions/checkout@v2

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 2.0.0.pre (Prerelease)
+* Drop support for Ruby `<2.7`.
+* Drop faraday dependencies.
+* Loosen version constraints on other dependencies.
+* Update measurometer metrics for consistency and clarity.
+
+## 1.7.0
+* Add support for `ARW` files.
+
+## 1.6.0
+* Add support for `AAC` files.
+
 ## 1.5.0
 * Add support for `NEF` files.
 

data/README.md

@@ -194,7 +194,7 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
 - `IMG_9266_*.tif` and all it's variations were created by the project maintainers
 
 ### ARW
-- ARW example is downloaded from http://www.rawsamples.ch/ and is Creative Common Licensed.
+- ARW examples are downloaded from http://www.rawsamples.ch/ and are Creative Common Licensed.
 
 ### ZIP
 - The .zip fixture files have been created by the project maintainers

data/format_parser.gemspec

@@ -30,17 +30,15 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'ks', '~> 0.0'
-  spec.add_dependency 'exifr', '~> 1', '>= 1.3.8'
-  spec.add_dependency 'id3tag', '~> 0.14', '>= 0.14.2'
-  spec.add_dependency 'faraday', '~> 0.13'
-  spec.add_dependency 'faraday_middleware', '~> 0.14'
-  spec.add_dependency 'measurometer', '~> 1'
+  spec.add_dependency 'exifr', '>= 1.3.8'
+  spec.add_dependency 'id3tag', '>= 0.14.2'
+  spec.add_dependency 'ks'
+  spec.add_dependency 'measurometer'
 
-  spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rake', '~> 12'
-  spec.add_development_dependency 'simplecov', '~> 0.15'
-  spec.add_development_dependency 'yard', '~> 0.9'
-  spec.add_development_dependency 'wetransfer_style', '0.5.0'
   spec.add_development_dependency 'parallel_tests'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'wetransfer_style', '1.0.0'
+  spec.add_development_dependency 'yard'
 end

data/lib/care.rb

@@ -96,12 +96,8 @@ class Care
     # @return [String, nil] the content read from the IO or `nil` if no data was available
     # @raise ArgumentError
     def byteslice(io, at, n_bytes)
-      if n_bytes < 1
-        raise ArgumentError, "The number of bytes to fetch must be a positive Integer, but was #{n_bytes}"
-      end
-      if at < 0
-        raise ArgumentError, "Negative offsets are not supported (got #{at})"
-      end
+      raise ArgumentError, "The number of bytes to fetch must be a positive Integer, but was #{n_bytes}" if n_bytes < 1
+      raise ArgumentError, "Negative offsets are not supported (got #{at})" if at < 0
 
       first_page = at / @page_size
       last_page = (at + n_bytes) / @page_size
@@ -174,16 +170,14 @@ class Care
     # @param io[IO] the IO to read from
     # @param page_i[Integer] which page (zero-based) to read
     def read_page(io, page_i)
-      Measurometer.increment_counter('format_parser.parser.Care.page_reads_from_upsteam', 1)
+      Measurometer.increment_counter('format_parser.parser.care.page_reads_from_upsteam', 1)
 
       io.seek(page_i * @page_size)
-      read_result = Measurometer.instrument('format_parser.Care.read_page') { io.read(@page_size) }
+      read_result = Measurometer.instrument('format_parser.care.read_page') { io.read(@page_size) }
       if read_result.nil?
         # If the read went past the end of the IO the read result will be nil,
         # so we know our IO is exhausted here
-        if @lowest_known_empty_page.nil? || @lowest_known_empty_page > page_i
-          @lowest_known_empty_page = page_i
-        end
+        @lowest_known_empty_page = page_i if @lowest_known_empty_page.nil? || @lowest_known_empty_page > page_i
       elsif read_result.bytesize < @page_size
         # If we read less than we initially wanted we know there are no pages
         # to read following this one, so we can also optimize

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '1.6.0'
+  VERSION = '2.0.0.pre'
 end

data/lib/format_parser.rb

@@ -20,6 +20,7 @@ module FormatParser
   require_relative 'care'
   require_relative 'active_storage/blob_analyzer'
   require_relative 'text'
+  require_relative 'string'
 
   # Define Measurometer in the internal namespace as well
   # so that we stay compatible for the applications that use it
@@ -87,8 +88,8 @@ module FormatParser
   # Parses the resource at the given `url` and returns the results as if it were any IO
   # given to `.parse`. The accepted keyword arguments are the same as the ones for `parse`.
   #
-  # @param url[String, URI] the HTTP(S) URL to request the object from using Faraday and `Range:` requests
-  # @param headers[Hash] (optional) the HTTP headers to request the object from using Faraday
+  # @param url[String, URI] the HTTP(S) URL to request the object from using `Range:` requests
+  # @param headers[Hash] (optional) the HTTP headers to request the object from
   # @param kwargs the keyword arguments to be delegated to `.parse`
   # @see {.parse}
   def self.parse_http(url, headers: {}, **kwargs)
@@ -177,9 +178,7 @@ module FormatParser
     # Convert the results from a lazy enumerator to an Array.
     results = results.to_a
 
-    if results.empty?
-      Measurometer.increment_counter('format_parser.unknown_files', 1)
-    end
+    Measurometer.increment_counter('format_parser.unknown_files', 1) if results.empty?
 
     amount == 1 ? results.first : results
   ensure
@@ -202,12 +201,12 @@ module FormatParser
   end
 
   def self.execute_parser_and_capture_expected_exceptions(parser, limited_io)
-    parser_name_for_instrumentation = parser.class.to_s.split('::').last
+    parser_name_for_instrumentation = parser.class.to_s.split('::').last.underscore
     Measurometer.instrument('format_parser.parser.%s' % parser_name_for_instrumentation) do
       parser.call(limited_io).tap do |result|
         if result
-          Measurometer.increment_counter('format_parser.detected_natures.%s' % result.nature, 1)
-          Measurometer.increment_counter('format_parser.detected_formats.%s' % result.format, 1)
+          Measurometer.increment_counter('format_parser.detected_natures', 1, nature: result.nature)
+          Measurometer.increment_counter('format_parser.detected_formats', 1, format: result.format)
         end
       end
     end
@@ -252,9 +251,7 @@ module FormatParser
     fitting_by_formats = assemble_parser_set[@parsers_per_format, desired_formats]
     parsers = fitting_by_natures & fitting_by_formats
 
-    if parsers.empty?
-      raise ArgumentError, "No parsers provide both natures #{desired_natures.inspect} and formats #{desired_formats.inspect}"
-    end
+    raise ArgumentError, "No parsers provide both natures #{desired_natures.inspect} and formats #{desired_formats.inspect}" if parsers.empty?
 
     # Order the parsers according to their priority value. The ones having a lower
     # value will sort higher and will be applied sooner

data/lib/io_utils.rb

@@ -9,12 +9,8 @@ module FormatParser::IOUtils
     raise ArgumentError, 'Unbounded reads are not supported' if n.nil?
     buf = io.read(n)
 
-    unless buf
-      raise InvalidRead, "We wanted to read #{n} bytes from the IO, but the IO is at EOF"
-    end
-    if buf.bytesize != n
-      raise InvalidRead, "We wanted to read #{n} bytes from the IO, but we got #{buf.bytesize} instead"
-    end
+    raise InvalidRead, "We wanted to read #{n} bytes from the IO, but the IO is at EOF" unless buf
+    raise InvalidRead, "We wanted to read #{n} bytes from the IO, but we got #{buf.bytesize} instead" if buf.bytesize != n
 
     buf
   end

data/lib/parsers/aac_parser/adts_header_info.rb

@@ -33,23 +33,17 @@ class FormatParser::AdtsHeaderInfo
   MPEG_VERSION_HASH = { 0 => 'MPEG-4', 1 => 'MPEG-2'}
 
   def mpeg4_sampling_frequency
-    if !@mpeg4_sampling_frequency_index.nil? && MPEG4_AUDIO_SAMPLING_FREQUENCY_HASH.key?(@mpeg4_sampling_frequency_index)
-      return MPEG4_AUDIO_SAMPLING_FREQUENCY_HASH[@mpeg4_sampling_frequency_index]
-    end
+    return MPEG4_AUDIO_SAMPLING_FREQUENCY_HASH[@mpeg4_sampling_frequency_index] if !@mpeg4_sampling_frequency_index.nil? && MPEG4_AUDIO_SAMPLING_FREQUENCY_HASH.key?(@mpeg4_sampling_frequency_index)
     nil
   end
 
   def profile_description
-    if !@profile.nil? && AAC_PROFILE_DESCRIPTION_HASH.key?(@profile)
-      return AAC_PROFILE_DESCRIPTION_HASH[@profile]
-    end
+    return AAC_PROFILE_DESCRIPTION_HASH[@profile] if !@profile.nil? && AAC_PROFILE_DESCRIPTION_HASH.key?(@profile)
     nil
   end
 
   def mpeg_version_description
-    if !@mpeg_version.nil? && MPEG_VERSION_HASH.key?(@mpeg_version)
-      return MPEG_VERSION_HASH[@mpeg_version]
-    end
+    return MPEG_VERSION_HASH[@mpeg_version] if !@mpeg_version.nil? && MPEG_VERSION_HASH.key?(@mpeg_version)
     nil
   end
 

data/lib/parsers/arw_parser.rb

@@ -0,0 +1,50 @@
+require_relative 'exif_parser'
+
+class FormatParser::ARWParser
+  include FormatParser::IOUtils
+  include FormatParser::EXIFParser
+
+  # Standard TIFF headers
+  MAGIC_LE = [0x49, 0x49, 0x2A, 0x0].pack('C4')
+  MAGIC_BE = [0x4D, 0x4D, 0x0, 0x2A].pack('C4')
+  HEADER_BYTES = [MAGIC_LE, MAGIC_BE]
+  ARW_MIME_TYPE = 'image/x-sony-arw'
+
+  def likely_match?(filename)
+    filename =~ /\.arw$/i
+  end
+
+  def call(io)
+    io = FormatParser::IOConstraint.new(io)
+
+    return unless HEADER_BYTES.include?(safe_read(io, 4))
+    exif_data = exif_from_tiff_io(io)
+
+    return unless valid?(exif_data)
+
+    w = exif_data.width || exif_data.pixel_x_dimension
+    h = exif_data.height || exif_data.pixel_y_dimension
+
+    FormatParser::Image.new(
+      format: :arw,
+      width_px: w,
+      height_px: h,
+      display_width_px: exif_data.rotated? ? h : w,
+      display_height_px: exif_data.rotated? ? w : h,
+      orientation: exif_data.orientation_sym,
+      intrinsics: { exif: exif_data },
+      content_type: ARW_MIME_TYPE,
+    )
+  rescue EXIFR::MalformedTIFF
+    nil
+  end
+
+  def valid?(exif_data)
+    # taken directly from tiff_parser.rb
+    # Similar to how exiftool determines the image type as ARW, we are implementing a check here
+    # https://github.com/exiftool/exiftool/blob/e969456372fbaf4b980fea8bb094d71033ac8bf7/lib/Image/ExifTool/Exif.pm#L929
+    exif_data.compression == 6 && exif_data.new_subfile_type == 1 && exif_data.make&.start_with?('SONY')
+  end
+
+  FormatParser.register_parser new, natures: :image, formats: :arw
+end

data/lib/parsers/dpx_parser/dpx_structs.rb

@@ -201,7 +201,7 @@ class FormatParser::DPXParser
     blanking :reserve, 52
 
     # Only expose the elements present
-    def image_elements #:nodoc:
+    def image_elements # :nodoc:
       @image_elements[0...number_elements]
     end
   end

data/lib/parsers/exif_parser.rb

@@ -125,9 +125,7 @@ module FormatParser::EXIFParser
       # those and return the _last_ non-0 orientation, or 0 otherwise
       @multiple_exif_results.reverse_each do |exif_tag_frame|
         orientation_value = exif_tag_frame.orientation
-        if !orientation_value.nil? && orientation_value != 0
-          return orientation_value
-        end
+        return orientation_value if !orientation_value.nil? && orientation_value != 0
       end
       0 # If none were found - the orientation is unknown
     end
@@ -175,7 +173,7 @@ module FormatParser::EXIFParser
   EXIFR.logger = Logger.new(nil)
 
   def exif_from_tiff_io(constrained_io, should_include_sub_ifds = false)
-    Measurometer.instrument('format_parser.EXIFParser.exif_from_tiff_io') do
+    Measurometer.instrument('format_parser.exif_parser.exif_from_tiff_io') do
       extended_io = IOExt.new(constrained_io)
       exif_raw_data = EXIFR::TIFF.new(extended_io)
 

data/lib/parsers/fdx_parser.rb

@@ -24,9 +24,9 @@ class FormatParser::FDXParser
   def check_for_document_type(file_and_document_type)
     sanitized_data = file_and_document_type.downcase
     if sanitized_data.include?('finaldraft') && sanitized_data.include?('script')
-      return :fdx, :script
+      [:fdx, :script]
     else
-      return
+      nil
     end
   end
 

data/lib/parsers/flac_parser.rb

@@ -20,15 +20,11 @@ class FormatParser::FLACParser
 
     minimum_block_size = bytestring_to_int(safe_read(io, 2))
 
-    if minimum_block_size < 16
-      raise MalformedFile, 'FLAC file minimum block size must be larger than 16'
-    end
+    raise MalformedFile, 'FLAC file minimum block size must be larger than 16' if minimum_block_size < 16
 
     maximum_block_size = bytestring_to_int(safe_read(io, 2))
 
-    if maximum_block_size < minimum_block_size
-      raise MalformedFile, 'FLAC file maximum block size must be equal to or larger than minimum block size'
-    end
+    raise MalformedFile, 'FLAC file maximum block size must be equal to or larger than minimum block size' if maximum_block_size < minimum_block_size
 
     minimum_frame_size = bytestring_to_int(safe_read(io, 3))
     maximum_frame_size = bytestring_to_int(safe_read(io, 3))

data/lib/parsers/jpeg_parser.rb

@@ -69,7 +69,7 @@ class FormatParser::JPEGParser
       end
     end
 
-    Measurometer.add_distribution_value('format_parser.JPEGParser.bytes_read_until_capture', @buf.pos)
+    Measurometer.add_distribution_value('format_parser.jpeg_parser.bytes_read_until_capture', @buf.pos)
 
     # A single file might contain multiple EXIF data frames. In a JPEG this would
     # manifest as multiple APP1 markers. The way different programs handle these
@@ -156,7 +156,7 @@ class FormatParser::JPEGParser
     # Use StringIO.new instead of #write - https://github.com/aws/aws-sdk-ruby/issues/785#issuecomment-95456838
     exif_buf = StringIO.new(safe_read(@buf, app1_frame_content_length - EXIF_MAGIC_STRING.bytesize))
 
-    Measurometer.add_distribution_value('format_parser.JPEGParser.bytes_sent_to_exif_parser', exif_buf.size)
+    Measurometer.add_distribution_value('format_parser.jpeg_parser.bytes_sent_to_exif_parser', exif_buf.size)
 
     @exif_data_frames << exif_from_tiff_io(exif_buf)
   rescue EXIFR::MalformedTIFF

data/lib/parsers/moov_parser.rb

@@ -37,7 +37,7 @@ class FormatParser::MOOVParser
     # size that gets parsed just before.
     max_read_offset = 0xFFFFFFFF
     decoder = Decoder.new
-    atom_tree = Measurometer.instrument('format_parser.Decoder.extract_atom_stream') do
+    atom_tree = Measurometer.instrument('format_parser.decoder.extract_atom_stream') do
       decoder.extract_atom_stream(io, max_read_offset)
     end
 
@@ -93,12 +93,10 @@ class FormatParser::MOOVParser
   def parse_dimensions(decoder, atom_tree)
     video_trak_atom = decoder.find_video_trak_atom(atom_tree)
 
-    tkhd = begin
-      if video_trak_atom
-        decoder.find_first_atom_by_path([video_trak_atom], 'trak', 'tkhd')
-      else
-        decoder.find_first_atom_by_path(atom_tree, 'moov', 'trak', 'tkhd')
-      end
+    tkhd = if video_trak_atom
+      decoder.find_first_atom_by_path([video_trak_atom], 'trak', 'tkhd')
+    else
+      decoder.find_first_atom_by_path(atom_tree, 'moov', 'trak', 'tkhd')
     end
 
     if tkhd

data/lib/parsers/mp3_parser.rb

@@ -179,13 +179,9 @@ class FormatParser::MP3Parser
         frame_data_str = io.read(frame_detail.frame_length)
         io.seek(io.pos - frame_detail.frame_length)
         xing_header = attempt_xing_header(frame_data_str)
-        if xing_header_usable_for_duration?(xing_header)
-          return [xing_header, mpeg_frames]
-        end
-      end
-      if frame_detail.frame_length > 1 # jump over current frame body
-        io.seek(io.pos + frame_detail.frame_length - bytes_to_read)
+        return [xing_header, mpeg_frames] if xing_header_usable_for_duration?(xing_header)
       end
+      io.seek(io.pos + frame_detail.frame_length - bytes_to_read) if frame_detail.frame_length > 1 # jump over current frame body
     end
     [nil, mpeg_frames]
   rescue InvalidDeepFetch # A frame was invalid - bail out since it's unlikely we can recover

data/lib/parsers/mpeg_parser.rb

@@ -44,9 +44,7 @@ class FormatParser::MPEGParser
       io.seek(pos + 1)
       horizontal_size, vertical_size = parse_image_size(io)
       ratio_code, rate_code = parse_rate_information(io)
-      if valid_aspect_ratio_code?(ratio_code) && valid_frame_rate_code?(rate_code)
-        return file_info(horizontal_size, vertical_size, ratio_code, rate_code)
-      end
+      return file_info(horizontal_size, vertical_size, ratio_code, rate_code) if valid_aspect_ratio_code?(ratio_code) && valid_frame_rate_code?(rate_code)
     end
     nil # otherwise the return value of Integer#times will be returned
   rescue FormatParser::IOUtils::InvalidRead

data/lib/parsers/tiff_parser.rb

@@ -6,7 +6,6 @@ class FormatParser::TIFFParser
   MAGIC_BE = [0x4D, 0x4D, 0x0, 0x2A].pack('C4')
   HEADER_BYTES = [MAGIC_LE, MAGIC_BE]
   TIFF_MIME_TYPE = 'image/tiff'
-  ARW_MIME_TYPE = 'image/x-sony-arw'
 
   def likely_match?(filename)
     filename =~ /\.tiff?$/i
@@ -28,20 +27,20 @@ class FormatParser::TIFFParser
     exif_data = exif_from_tiff_io(io)
     return unless exif_data
 
+    return if arw?(exif_data)
+
     w = exif_data.width || exif_data.pixel_x_dimension
     h = exif_data.height || exif_data.pixel_y_dimension
 
-    format = arw?(exif_data) ? :arw : :tif
-    mime_type = arw?(exif_data) ? ARW_MIME_TYPE : TIFF_MIME_TYPE
     FormatParser::Image.new(
-      format: format,
+      format: :tif,
       width_px: w,
       height_px: h,
       display_width_px: exif_data.rotated? ? h : w,
       display_height_px: exif_data.rotated? ? w : h,
       orientation: exif_data.orientation_sym,
       intrinsics: {exif: exif_data},
-      content_type: mime_type,
+      content_type: TIFF_MIME_TYPE,
     )
   rescue EXIFR::MalformedTIFF
     nil
@@ -55,7 +54,7 @@ class FormatParser::TIFFParser
   # Similar to how exiftool determines the image type as ARW, we are implementing a check here
   # https://github.com/exiftool/exiftool/blob/e969456372fbaf4b980fea8bb094d71033ac8bf7/lib/Image/ExifTool/Exif.pm#L929
   def arw?(exif_data)
-    exif_data.compression == 6 && exif_data.new_subfile_type == 1 && exif_data.make == 'SONY'
+    exif_data.compression == 6 && exif_data.new_subfile_type == 1 && exif_data.make&.start_with?('SONY')
   end
 
   FormatParser.register_parser new, natures: :image, formats: :tif

data/lib/parsers/wav_parser.rb

@@ -34,9 +34,7 @@ class FormatParser::WAVParser
       case chunk_type
       when 'fmt ' # watch out: the chunk ID of the format chunk ends with a space
         fmt_data = unpack_fmt_chunk(io, chunk_size)
-        if fmt_data[:audio_format] != 1 and fact_processed
-          return process_non_pcm(fmt_data, total_sample_frames)
-        end
+        return process_non_pcm(fmt_data, total_sample_frames) if fmt_data[:audio_format] != 1 and fact_processed
         fmt_processed = true
       when 'data'
         return unless fmt_processed # the 'data' chunk cannot preceed the 'fmt ' chunk
@@ -45,11 +43,10 @@ class FormatParser::WAVParser
       when 'fact'
         total_sample_frames = safe_read(io, 4).unpack('l').first
         safe_skip(io, chunk_size - 4)
-        if fmt_processed and fmt_data[:audio_format] != 1
-          return process_non_pcm(fmt_data, total_sample_frames)
-        end
+        return process_non_pcm(fmt_data, total_sample_frames) if fmt_processed and fmt_data[:audio_format] != 1
         fact_processed = true
-      else # Skip this chunk until a known chunk is encountered
+      else
+        # Skip this chunk until a known chunk is encountered
         safe_skip(io, chunk_size)
       end
     end
@@ -70,11 +67,11 @@ class FormatParser::WAVParser
     safe_skip(io, chunk_size - 16) # skip the extra fields
 
     {
-      audio_format:    fmt_info[0],
-      channels:        fmt_info[1],
-      sample_rate:     fmt_info[2],
-      byte_rate:       fmt_info[3],
-      block_align:     fmt_info[4],
+      audio_format: fmt_info[0],
+      channels: fmt_info[1],
+      sample_rate: fmt_info[2],
+      byte_rate: fmt_info[3],
+      block_align: fmt_info[4],
       bits_per_sample: fmt_info[5],
     }
   end