format_parser 0.29.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8efef8742cfefdcde0aa05ab32c726004325c2210b19e25ca2cf866ad786074
-  data.tar.gz: bbd59f6f046a35cff93ea4e20a5168c2912007b06b3fabe7a5b23f7a0ed70c34
+  metadata.gz: e49394510e191185e3be1f3560ca182a103b9446225a1163294f2db6a912ae76
+  data.tar.gz: 4f52d761857020a0957711528ad4d10ee885c74745897fa14a5e69292abf0cc2
 SHA512:
-  metadata.gz: b4a7d965be6ff0cb0abf8cdec5f4d31259aada155ae465c1b31d5eba718f9d0071ee8b7a6b2061e6024add45d32b901c0f95a37373727a59fd4a6562041f52e9
-  data.tar.gz: d314cb6876185ddedcef01c67205cf0635c2d8aebd0871dfd7834f0f54c15c38c31db7cc730c6aea8f8a72aff85949fa6937d970dfb320120a5b8c843f003753
+  metadata.gz: 829bd488d021cb0db5ce2b718aa661bdf3d4f86a55e55808c8279367e955d35b2bdccdd2bdf805ff736cef6d49f8c0f425c1ec65c0d7f07ada2db684581d4c1f
+  data.tar.gz: ed76f04c157a865de03df0d614d0d353b53e1e9c9b0a164ac6b2a6612a358643ae210b00fd92b91fec6cefd8c8c71e6ff559e63df3343b13ce161175425dbd3a

data/.github/workflows/main.yml

@@ -16,9 +16,6 @@ jobs:
           - 2.7
           - 2.6
           - 2.5
-          - 2.4
-          - 2.3
-          - 2.2
           - jruby
     steps:
       - name: Checkout
@@ -65,12 +62,11 @@ jobs:
           - 2.7
           - 2.6
           - 2.5
-          - 2.4
-          - 2.3
-          - 2.2
           - jruby
         experimental: [false]
         include:
+          - ruby: 3.1
+            experimental: true
           - ruby: 3.0
             experimental: true
     steps:

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 1.1.0
+* Add support for `frame_rate` in moov_parser
+
+## 1.0.0
+* Dropping support for Ruby 2.2.X, 2.3.X and 2.4.X
+* MP3: Fix negative length reads in edge cases by bumping `id3tag` version to `v0.14.2`
+
+## 0.29.1
+* Fix handling of 200 responses with `parse_http` as well as handling of very small responses which do not need range access
+
 ## 0.29.0
 * Add option `headers:` to `FormatParser.parse_http`
 

data/Gemfile

@@ -1,4 +1,8 @@
 source 'https://rubygems.org'
 
+gem 'ruby-debug-ide'
+gem 'debase'
+gem 'solargraph', group: :development
+
 # Gem dependencies specified in the gemspec
 gemspec

data/format_parser.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'ks', '~> 0.0'
   spec.add_dependency 'exifr', '~> 1', '>= 1.3.8'
-  spec.add_dependency 'id3tag', '~> 0.14'
+  spec.add_dependency 'id3tag', '~> 0.14', '>= 0.14.2'
   spec.add_dependency 'faraday', '~> 0.13'
   spec.add_dependency 'faraday_middleware', '~> 0.14'
   spec.add_dependency 'measurometer', '~> 1'

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.29.0'
+  VERSION = '1.1.0'
 end

data/lib/parsers/moov_parser/decoder.rb

@@ -72,6 +72,7 @@ class FormatParser::MOOVParser::Decoder
 
     trak_atoms.find do |trak_atom|
       hdlr_atom = find_first_atom_by_path([trak_atom], 'trak', 'mdia', 'hdlr')
+      next if hdlr_atom.nil?
       hdlr_atom.atom_fields[:component_type] == 'mhlr' && hdlr_atom.atom_fields[:component_subtype] == 'vide'
     end
   end
@@ -111,6 +112,24 @@ class FormatParser::MOOVParser::Decoder
     }
   end
 
+  def parse_stts_atom(io, _)
+    version = read_byte_value(io)
+    is_v1 = version == 1
+    stts = {
+      version: version,
+      flags: read_bytes(io, 3),
+      number_of_entries: is_v1 ? read_64bit_uint(io) : read_32bit_uint(io),
+      entries: []
+    }
+    stts[:number_of_entries].times {
+      stts[:entries] << {
+        sample_count: read_32bit_uint(io),
+        sample_duration: read_32bit_uint(io)
+      }
+    }
+    stts
+  end
+
   def parse_mdhd_atom(io, _)
     version = read_byte_value(io)
     is_v1 = version == 1

data/lib/parsers/moov_parser.rb

@@ -48,9 +48,9 @@ class FormatParser::MOOVParser
     width, height = parse_dimensions(decoder, atom_tree)
 
     # Try to find the "topmost" duration (respecting edits)
-    if mdhd = decoder.find_first_atom_by_path(atom_tree, 'moov', 'mvhd')
-      timescale = mdhd.field_value(:tscale)
-      duration = mdhd.field_value(:duration)
+    if mvhd = decoder.find_first_atom_by_path(atom_tree, 'moov', 'mvhd')
+      timescale = mvhd.field_value(:tscale)
+      duration = mvhd.field_value(:duration)
       media_duration_s = duration / timescale.to_f
     end
 
@@ -64,13 +64,15 @@ class FormatParser::MOOVParser
         intrinsics: atom_tree,
       )
     else
+      frame_rate = parse_sample_atom(decoder, atom_tree)&.truncate(2)
       FormatParser::Video.new(
         format: format_from_moov_type(file_type),
         width_px: width,
         height_px: height,
+        frame_rate: frame_rate,
         media_duration_seconds: media_duration_s,
         content_type: MP4_MIXED_MIME_TYPE,
-        intrinsics: atom_tree,
+        intrinsics: atom_tree
       )
     end
   end
@@ -115,5 +117,35 @@ class FormatParser::MOOVParser
     maybe_atom_size >= minimum_ftyp_atom_size && maybe_ftyp_atom_signature == 'ftyp'
   end
 
+  # Sample information is found in the 'time-to-sample' stts atom.
+  # The media atom mdhd is needed too in order to get the movie timescale
+  def parse_sample_atom(decoder, atom_tree)
+    video_trak_atom = decoder.find_video_trak_atom(atom_tree)
+
+    stts = if video_trak_atom
+      decoder.find_first_atom_by_path([video_trak_atom], 'trak', 'mdia', 'minf', 'stbl', 'stts')
+    else
+      decoder.find_first_atom_by_path(atom_tree, 'moov', 'trak', 'mdia', 'minf', 'stbl', 'stts')
+    end
+
+    mdhd = if video_trak_atom
+      decoder.find_first_atom_by_path([video_trak_atom], 'trak', 'mdia', 'mdhd')
+    else
+      decoder.find_first_atom_by_path(atom_tree, 'moov', 'trak', 'mdia', 'mdhd')
+    end
+
+    if stts && mdhd
+      timescale = mdhd.atom_fields[:tscale]
+      sample_duration = stts.field_value(:entries).first[:sample_duration]
+      if timescale.nil? || timescale == 0 || sample_duration.nil? || sample_duration == 0
+        nil
+      else
+        timescale.to_f / sample_duration
+      end
+    else
+      nil
+    end
+  end
+
   FormatParser.register_parser new, natures: :video, formats: FTYP_MAP.values, priority: 1
 end

data/lib/remote_io.rb

@@ -89,18 +89,32 @@ class FormatParser::RemoteIO
     response = conn.get(@uri, nil, range: 'bytes=%d-%d' % [range.begin, range.end])
 
     case response.status
-    when 200, 206
+    when 200
+      # S3 returns 200 when you request a Range that is fully satisfied by the entire object,
+      # we take that into account here. Also, for very tiny responses (and also for empty responses)
+      # the responses are going to be 200 which does not mean we cannot proceed
+      # To have a good check for both of these conditions we need to know whether the ranges overlap fully
+      response_size = response.body.bytesize
+      requested_range_size = range.end - range.begin + 1
+      if response_size > requested_range_size
+        error_message = [
+          "We requested #{requested_range_size} bytes, but the server sent us more",
+          "(#{response_size} bytes) - it likely has no `Range:` support.",
+          "The error occurred when talking to #{@uri})"
+        ]
+        raise InvalidRequest.new(response.status, error_message.join("\n"))
+      end
+      [response_size, response.body]
+    when 206
       # Figure out of the server supports content ranges, if it doesn't we have no
       # business working with that server
       range_header = response.headers['Content-Range']
-      raise InvalidRequest.new(response.status, "No range support at #{@uri}") unless range_header
+      raise InvalidRequest.new(response.status, "The server replied with 206 status but no Content-Range at #{@uri}") unless range_header
 
       # "Content-Range: bytes 0-0/307404381" is how the response header is structured
       size = range_header[/\/(\d+)$/, 1].to_i
 
-      # S3 returns 200 when you request a Range that is fully satisfied by the entire object,
-      # we take that into account here. For other servers, 206 is the expected response code.
-      # Also, if we request a _larger_ range than what can be satisfied by the server,
+      # If we request a _larger_ range than what can be satisfied by the server,
       # the response is going to only contain what _can_ be sent and the status is also going
       # to be 206
       return [size, response.body]

data/lib/video.rb

@@ -8,6 +8,8 @@ module FormatParser
 
     attr_accessor :height_px
 
+    attr_accessor :frame_rate
+
     # Type of the file (e.g :mp3)
     attr_accessor :format
 

data/spec/integration/active_storage/rails_app.rb

@@ -53,19 +53,22 @@ end
 require 'minitest/autorun'
 require 'open-uri'
 
+fixtures_dir = File.join(File.dirname(__FILE__), '../../fixtures')
+
 describe User do
   describe "profile_picture's metadatas" do
     it 'parse metadatas with format_parser' do
+      fixture_path = fixtures_dir + '/PNG/cat.png'
       user = User.create
       user.profile_picture.attach(
         filename: 'cat.png',
-        io: URI.open('https://freesvg.org/img/1416155153.png')
+        io: File.open(fixture_path, 'rb')
       )
 
       user.profile_picture.analyze
 
-      _(user.profile_picture.metadata[:width_px]).must_equal 500
-      _(user.profile_picture.metadata[:height_px]).must_equal 296
+      _(user.profile_picture.metadata[:width_px]).must_equal 600
+      _(user.profile_picture.metadata[:height_px]).must_equal 600
       _(user.profile_picture.metadata[:color_mode]).must_equal 'rgba'
     end
   end

data/spec/parsers/moov_parser_spec.rb

@@ -106,6 +106,7 @@ describe FormatParser::MOOVParser do
     expect(result.format).to eq(:mov)
     expect(result.width_px).to eq(160)
     expect(result.height_px).to eq(90)
+    expect(result.frame_rate).to eq(14.98)
   end
 
   it 'provides filename hints' do
@@ -122,6 +123,7 @@ describe FormatParser::MOOVParser do
     expect(result.format).to eq(:mov)
     expect(result.width_px).to eq(640)
     expect(result.height_px).to eq(360)
+    expect(result.frame_rate).to eq(30)
   end
 
   it 'does not raise error when a meta atom has size 0' do

data/spec/remote_fetching_spec.rb

@@ -19,18 +19,27 @@ describe 'Fetching data from HTTP remotes' do
       res.status = 302
       res.header['Location'] = req.path.sub('/redirect', '')
     end
+    @server.mount_proc '/empty' do |_req, res|
+      res.status = 200
+      res.body = ''
+    end
+    @server.mount_proc '/tiny' do |_req, res|
+      res.status = 200
+      res.body = File.read(fixtures_dir + '/test.gif')
+    end
+
     trap('INT') { @server.stop }
     @server_thread = Thread.new { @server.start }
   end
 
-  it '#parse_http is called without any option' do
+  it 'works with .parse_http called without any options' do
     result = FormatParser.parse_http('http://localhost:9399/PNG/anim.png')
 
     expect(result.format).to eq(:png)
     expect(result.height_px).to eq(180)
   end
 
-  it '#parse_http is called with hash options' do
+  it 'works with .parse_http called with additional options' do
     fake_result = double(nature: :audio, format: :aiff)
     expect_any_instance_of(FormatParser::AIFFParser).to receive(:call).and_return(fake_result)
     results = FormatParser.parse_http('http://localhost:9399/PNG/anim.png', results: :all)
@@ -39,6 +48,18 @@ describe 'Fetching data from HTTP remotes' do
     expect(results).to include(fake_result)
   end
 
+  it 'is able to cope with a 0-size resource which does not provide Content-Range' do
+    file_information = FormatParser.parse_http('http://localhost:9399/empty')
+
+    expect(file_information).to be_nil
+  end
+
+  it 'is able to cope with a tiny resource which fits into the first requested range completely' do
+    file_information = FormatParser.parse_http('http://localhost:9399/tiny')
+    expect(file_information).not_to be_nil
+    expect(file_information.nature).to eq(:image)
+  end
+
   it 'parses the animated PNG over HTTP' do
     file_information = FormatParser.parse_http('http://localhost:9399/PNG/anim.png')
     expect(file_information).not_to be_nil

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.29.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Noah Berman
 - Julik Tarkhanov
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-18 00:00:00.000000000 Z
+date: 2022-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -52,6 +52,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.14'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -59,6 +62,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.14'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.2
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -294,7 +300,7 @@ licenses:
 - MIT (Hippocratic)
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -309,8 +315,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key:
+rubygems_version: 3.3.4
+signing_key: 
 specification_version: 4
 summary: A library for efficient parsing of file metadata
 test_files: []