format_parser 0.20.1 → 0.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d10f78d5d1a472fc6af704132ac8b98542a141de3d5b8a998faebfa5a994b30
-  data.tar.gz: 65636e308c67d8ccc3e3ff592e5cfe02bf5c105ac32b85d17825279211b362db
+  metadata.gz: ad2245de4a2119c7572c3962ad14abbf16395b2bec4064b218ee9f99d1e7c24b
+  data.tar.gz: b982bcc7f6626b66684db532317b0c0d35cd062aa89766ea1a230f93e7d996d6
 SHA512:
-  metadata.gz: bec31abec6687c7b8dd57783da5e2aec175e1f538097b03eba095c49bd636fc38d562b21e3da1b2ca24f74caffb37b9953e139372ce30be19aea502947b532ca
-  data.tar.gz: 51e593f4c14049467f657e786a2c9d54f91713d1ab45badbdd67d27876e1b61f3542936cb184cb72f0885745673b8423ce4ceda5c0b5cdb4661f96ebf2f6d59e
+  metadata.gz: 2df2a3763e12e2bb0c70a8f5ec3319fcf6a3210a73461dc8abb5ec2af706028403eca48cf8c589bd40122dff0cfdabce383a79b1253237e219a5c89936ec0a5e
+  data.tar.gz: 0ca6084649313b2c7ad32204b4c8b745f13dbf6cf2c347cb7c80ca7976b964f00feabd9666c0372e9a6c10e4ed250500ed4e202cdb687372cbd369378b7a0faa

data/.gitignore

@@ -54,3 +54,7 @@ Gemfile.lock
 
 # Used by RuboCop. Remote config files pulled in from inherit_from directive.
 # .rubocop-https?--*
+
+
+# OSX Files
+.DS_Store

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.21.0
+* Adds support for MPEG video files
+
 ## 0.20.1
 * Make sure EXIF results work correctly with ActiveSupport JSON encoders
 

data/README.md

@@ -31,6 +31,7 @@ and [dimensions,](https://github.com/sstephenson/dimensions) borrowing from them
 * ZIP
 * DOCX, PPTX, XLSX
 * OGG
+* MPEG, MPG
 
 ...with [more](https://github.com/WeTransfer/format_parser/issues?q=is%3Aissue+is%3Aopen+label%3Aformats) on the way!
 
@@ -173,6 +174,10 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
 ### .docx
 - The .docx files were generated by the project maintainers
 
+### .mpg and .mpeg
+- The files (video 1 to 4) were downloaded from https://standaloneinstaller.com/blog/big-list-of-sample-videos-for-testers-124.html.
+- Video 5 was downloaded from https://archive.org/details/ligouHDR-HC1_sample1.
+
 ### JPEG examples of EXIF  orientation
 - Downloaded from Unspash (and thus freely avaliable) - https://unsplash.com/license and have then been
   manipulated using the [https://github.com/recurser/exif-orientation-examples](exif-orientation-examples)

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.20.1'
+  VERSION = '0.21.0'
 end

data/lib/parsers/mpeg_parser.rb

@@ -0,0 +1,131 @@
+
+# MPEG Headers documentation:
+# http://dvd.sourceforge.net/dvdinfo/mpeghdrs.html#seq
+# http://www.cs.columbia.edu/~delbert/docs/Dueck%20--%20MPEG-2%20Video%20Transcoding.pdf
+# Useful tool to check the file information: https://www.metadata2go.com/
+class FormatParser::MPEGParser
+  extend FormatParser::IOUtils
+
+  ASPECT_RATIOS = {
+    1 => '1:1',
+    2 => '4:3',
+    3 => '16:9',
+    4 => '2.21:1'
+  }
+
+  FRAME_RATES = {
+    1 => '23.976',
+    2 => '24',
+    3 => '25',
+    4 => '29.97',
+    5 => '30',
+    6 => '50',
+    7 => '59.94',
+    8 => '60'
+  }
+
+  PACK_HEADER_START_CODE = [0x00, 0x00, 0x01, 0xBA].pack('C*')
+  SEQUENCE_HEADER_START_CODE = [0xB3].pack('C*')
+  SEEK_FOR_SEQUENCE_HEADER_TIMES_LIMIT = 4
+  SEEK_FOR_SEQUENCE_HEADER_START_CODE_TIMES_LIMIT = 4
+  BYTES_TO_READ_PER_TIME = 1024
+
+  def self.likely_match?(filename)
+    filename =~ /\.(mpg|mpeg)$/i
+  end
+
+  def self.call(io)
+    return unless matches_mpeg_header?(io)
+
+    # We are looping though the stream because there can be several sequence headers and some of them are not usefull.
+    # If we detect that the header is not usefull, then we look for the next one for SEEK_FOR_SEQUENCE_HEADER_TIMES_LIMIT
+    # If we reach the EOF, then the mpg is likely to be corrupted and we return nil
+    SEEK_FOR_SEQUENCE_HEADER_TIMES_LIMIT.times do
+      return if fetch_next_sequence_header_code_position(io).nil?
+      horizontal_size, vertical_size = parse_image_size(io)
+      ratio_code, rate_code = parse_rate_information(io)
+
+      if valid_aspect_ratio_code?(ratio_code) && valid_frame_rate_code?(rate_code)
+        return file_info(horizontal_size, vertical_size, ratio_code, rate_code)
+      end
+    end
+  rescue FormatParser::IOUtils::InvalidRead
+    nil
+  end
+
+  def self.file_info(width_px, height_px, ratio_code, rate_code)
+    FormatParser::Video.new(
+      format: :mpg,
+      width_px: width_px,
+      height_px: height_px,
+      intrinsics: {
+        aspect_ratio: ASPECT_RATIOS.fetch(ratio_code),
+        frame_rate: FRAME_RATES.fetch(rate_code)
+      },
+    )
+  end
+
+  # The following 3 bytes after the sequence header code, gives us information about the px size
+  # 1.5 bytes (12 bits) for horizontal size and 1.5 bytes for vertical size
+  def self.parse_image_size(io)
+    image_size = convert_3_bytes_to_bits(safe_read(io, 3))
+    [read_first_12_bits(image_size), read_last_12_bits(image_size)]
+  end
+
+  # The following byte gives us information about the aspect ratio and frame rate
+  # 4 bits corresponds to the aspect ratio and 4 bits to the frame rate code
+  def self.parse_rate_information(io)
+    rate_information = safe_read(io, 1).unpack('C').first
+    [read_first_4_bits(rate_information), read_last_4_bits(rate_information)]
+  end
+
+  def self.valid_aspect_ratio_code?(ratio_code)
+    ASPECT_RATIOS.include?(ratio_code)
+  end
+
+  def self.valid_frame_rate_code?(rate_code)
+    FRAME_RATES.include?(rate_code)
+  end
+
+  # Returns the position of the next sequence package content in the stream
+  # This method will read BYTES_TO_READ_PER_TIME in each loop for a maximum amount of SEEK_FOR_SEQUENCE_HEADER_START_CODE_TIMES_LIMIT times
+  # If the package is not found, then it returns nil.
+  def self.fetch_next_sequence_header_code_position(io)
+    SEEK_FOR_SEQUENCE_HEADER_START_CODE_TIMES_LIMIT.times do
+      bytes_stream_read = io.read(BYTES_TO_READ_PER_TIME)
+      header_relative_index = bytes_stream_read.index(SEQUENCE_HEADER_START_CODE)
+      next if header_relative_index.nil?
+      new_io_pos = io.pos - BYTES_TO_READ_PER_TIME + header_relative_index + 1
+      io.seek(new_io_pos)
+      return new_io_pos
+    end
+  end
+
+  # If the first 4 bytes of the stream are equal to 00 00 01 BA, the pack start code for the Pack Header, then it's an MPEG file.
+  def self.matches_mpeg_header?(io)
+    safe_read(io, 4) == PACK_HEADER_START_CODE
+  end
+
+  def self.convert_3_bytes_to_bits(bytes)
+    bytes = bytes.unpack('CCC')
+    (bytes[0] << 16) | (bytes[1] << 8) | (bytes[2])
+  end
+
+  def self.read_first_12_bits(bits)
+    bits >> 12 & 0x0fff
+  end
+
+  def self.read_last_12_bits(bits)
+    bits & 0x0fff
+  end
+
+  def self.read_first_4_bits(byte)
+    byte >> 4
+  end
+
+  def self.read_last_4_bits(byte)
+    byte & 0x0F
+  end
+
+  FormatParser.register_parser self, natures: [:video], formats: [:mpg, :mpeg]
+end

data/spec/parsers/mpeg_parser_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+describe FormatParser::MPEGParser do
+  it 'parses a first example mpg file' do
+    parse_result = described_class.call(File.open(__dir__ + '/../fixtures/MPG/video1.mpg', 'rb'))
+
+    expect(parse_result.nature).to eq(:video)
+    expect(parse_result.format).to eq(:mpg)
+    expect(parse_result.width_px).to eq(560)
+    expect(parse_result.height_px).to eq(320)
+    expect(parse_result.intrinsics[:aspect_ratio]).to eq('1:1')
+    expect(parse_result.intrinsics[:frame_rate]).to eq('30')
+  end
+
+  it 'parses a file with mpeg extension' do
+    parse_result = described_class.call(File.open(__dir__ + '/../fixtures/MPG/video2.mpeg', 'rb'))
+
+    expect(parse_result.nature).to eq(:video)
+    expect(parse_result.format).to eq(:mpg)
+    expect(parse_result.width_px).to eq(720)
+    expect(parse_result.height_px).to eq(480)
+    expect(parse_result.intrinsics[:aspect_ratio]).to eq('4:3')
+    expect(parse_result.intrinsics[:frame_rate]).to eq('29.97')
+  end
+
+  it 'parses a second example mpg file' do
+    parse_result = described_class.call(File.open(__dir__ + '/../fixtures/MPG/video3.mpg', 'rb'))
+
+    expect(parse_result.nature).to eq(:video)
+    expect(parse_result.format).to eq(:mpg)
+    expect(parse_result.width_px).to eq(720)
+    expect(parse_result.height_px).to eq(496)
+    expect(parse_result.intrinsics[:aspect_ratio]).to eq('4:3')
+    expect(parse_result.intrinsics[:frame_rate]).to eq('29.97')
+  end
+
+  it 'parses a bigger mpg file' do
+    parse_result = described_class.call(File.open(__dir__ + '/../fixtures/MPG/video4.mpg', 'rb'))
+
+    expect(parse_result.nature).to eq(:video)
+    expect(parse_result.format).to eq(:mpg)
+    expect(parse_result.width_px).to eq(1920)
+    expect(parse_result.height_px).to eq(1080)
+    expect(parse_result.intrinsics[:aspect_ratio]).to eq('16:9')
+    expect(parse_result.intrinsics[:frame_rate]).to eq('29.97')
+  end
+
+  it 'parses a file with different malformed first sequence header' do
+    parse_result = described_class.call(File.open(__dir__ + '/../fixtures/MPG/video5.mpg', 'rb'))
+
+    expect(parse_result.nature).to eq(:video)
+    expect(parse_result.format).to eq(:mpg)
+    expect(parse_result.width_px).to eq(1440)
+    expect(parse_result.height_px).to eq(1080)
+    expect(parse_result.intrinsics[:aspect_ratio]).to eq('16:9')
+    expect(parse_result.intrinsics[:frame_rate]).to eq('25')
+  end
+
+  it 'parses a MP4 file' do
+    parse_result = described_class.call(File.open(__dir__ + '/../fixtures/MOOV/MP4/bmff.mp4', 'rb'))
+
+    expect(parse_result).to be_nil
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.20.1
+  version: 0.21.0
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-05 00:00:00.000000000 Z
+date: 2020-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks
@@ -240,6 +240,7 @@ files:
 - lib/parsers/moov_parser/decoder.rb
 - lib/parsers/mp3_parser.rb
 - lib/parsers/mp3_parser/id3_extraction.rb
+- lib/parsers/mpeg_parser.rb
 - lib/parsers/ogg_parser.rb
 - lib/parsers/pdf_parser.rb
 - lib/parsers/png_parser.rb
@@ -271,6 +272,7 @@ files:
 - spec/parsers/jpeg_parser_spec.rb
 - spec/parsers/moov_parser_spec.rb
 - spec/parsers/mp3_parser_spec.rb
+- spec/parsers/mpeg_parser_spec.rb
 - spec/parsers/ogg_parser_spec.rb
 - spec/parsers/pdf_parser_spec.rb
 - spec/parsers/png_parser_spec.rb
@@ -303,7 +305,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A library for efficient parsing of file metadata