format_parser 0.25.4 → 0.28.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 379af5330f9b9521b278e5fe54821398ea2a5a58d4ac9881bcc5f30021446c2a
-  data.tar.gz: 71da3110186c0c8fce731fc2adb05fbcbcc829ab9c6fe67b4934dc8d69fa093c
+  metadata.gz: d2212d92c3c0ca7aab0e60c9aa6559d0508ec2b8b39e4b2067990d97a7d78327
+  data.tar.gz: 27736f9d52ea1e73a147d96d38608e268040648e61d2b913ca38f164f2d2d0b6
 SHA512:
-  metadata.gz: 5a9a99547baa58e3e693c8e24cc84af0d1e6598403dbed9473b885531e7c067b51c5aac4cc411ddc3f7d114b7987b470eb81554422553e6b346fc42b98a50789
-  data.tar.gz: 851b0c4fad434140e641077c01e6d46f463c8aab7b85d35a85ba0729c2913bd978a35e7ab0608d4cb8d5e2e2f13761a00078b98948290c3b3203953a9cba7c03
+  metadata.gz: 5f63f74444df32d016fc25dc5c3731671690185b7ecf80ffc06da350b37d82b566f451508091245fec255c816121a115286dd04fb626aa4c1b694836fd645d97
+  data.tar.gz: ee1917287bb7c50bd71d9a7756321e30fbc73aa5e3a4f42e4844eb9494dfe77e89a9a26622732772c9b29c4ea673f3401324d2e6bf67e5bdaa5dac991d12f821

data/.github/workflows/main.yml

@@ -0,0 +1,104 @@
+name: CI
+
+on: [push,pull_request]
+
+env:
+  BUNDLE_PATH: vendor/bundle
+
+jobs:
+  lint:
+    name: Code Style
+    runs-on: ubuntu-18.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    strategy:
+      matrix:
+        ruby:
+          - 2.7
+          - 2.6
+          - 2.5
+          - 2.4
+          - 2.3
+          - 2.2
+          - jruby
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Gemfile Cache
+        uses: actions/cache@v2
+        with:
+          path: Gemfile.lock
+          key: ${{ runner.os }}-gemlock-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'format_parser.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gemlock-${{ matrix.ruby }}-
+      - name: Bundle Cache
+        id: cache-gems
+        uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'Gemfile.lock', 'format_parser.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-${{ matrix.ruby }}-
+            ${{ runner.os }}-gems-
+      - name: Bundle Install
+        if: steps.cache-gems.outputs.cache-hit != 'true'
+        run: bundle install --jobs 4 --retry 3
+      - name: Rubocop Cache
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/rubocop_cache
+          key: ${{ runner.os }}-rubocop-${{ hashFiles('.rubocop.yml') }}
+          restore-keys: |
+            ${{ runner.os }}-rubocop-
+      - name: Rubocop
+        run: bundle exec rubocop
+  test:
+    name: Specs
+    runs-on: ubuntu-18.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    strategy:
+      matrix:
+        ruby:
+          - 2.7
+          - 2.6
+          - 2.5
+          - 2.4
+          - 2.3
+          - 2.2
+          - jruby
+        experimental: [false]
+        include:
+          - ruby: 3.0
+            experimental: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Gemfile Cache
+        uses: actions/cache@v2
+        with:
+          path: Gemfile.lock
+          key: ${{ runner.os }}-gemlock-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'format_parser.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gemlock-${{ matrix.ruby }}-
+      - name: Bundle Cache
+        id: cache-gems
+        uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'Gemfile.lock', 'format_parser.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-${{ matrix.ruby }}-
+            ${{ runner.os }}-gems-
+      - name: Bundle Install
+        if: steps.cache-gems.outputs.cache-hit != 'true'
+        run: bundle install --jobs 4 --retry 3
+      - name: RSpec
+        continue-on-error: ${{ matrix.experimental }}
+        run: bundle exec rake parallel:spec

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 0.28.0
+* Change `FormatParser.parse_http` to follow HTTP redirects
+
+## 0.27.0
+* Add `#content_type` on `Result` return values which makes sense for the detected filetype
+
+## 0.26.0
+* Add support for M3U format files
+
+## 0.25.6
+* Fix FormatParser.parse (with `results: :first`) to be deterministic
+
+## 0.25.5
+* DPX: Fix DPXParser to support images without aspect ratio
+
 ## 0.25.4
 * MP3: Fix MP3Parser to return nil for TIFF files
 * Add support to ruby 2.7

data/README.md

@@ -32,6 +32,7 @@ and [dimensions,](https://github.com/sstephenson/dimensions) borrowing from them
 * DOCX, PPTX, XLSX
 * OGG
 * MPEG, MPG
+* M3U
 
 ...with [more](https://github.com/WeTransfer/format_parser/issues?q=is%3Aissue+is%3Aopen+label%3Aformats) on the way!
 
@@ -194,6 +195,9 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
   manipulated using the [https://github.com/recurser/exif-orientation-examples](exif-orientation-examples)
   script.
 
+### M3U
+- The M3U fixture files were created by one of the project maintainers
+
 ### .key
 - The `keynote_recognized_as_jpeg.key` file was created by the project maintainers
 

data/format_parser.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'exifr', '~> 1', '>= 1.3.8'
   spec.add_dependency 'id3tag', '~> 0.14'
   spec.add_dependency 'faraday', '~> 0.13'
+  spec.add_dependency 'faraday_middleware', '~> 0.14'
   spec.add_dependency 'measurometer', '~> 1'
 
   spec.add_development_dependency 'rspec', '~> 3.0'

data/lib/archive.rb

@@ -26,6 +26,9 @@ module FormatParser
     # it can be placed here
     attr_accessor :intrinsics
 
+    # The MIME type of the archive
+    attr_accessor :content_type
+
     # Only permits assignments via defined accessors
     def initialize(**attributes)
       attributes.map { |(k, v)| public_send("#{k}=", v) }

data/lib/audio.rb

@@ -35,6 +35,9 @@ module FormatParser
     # it can be placed here
     attr_accessor :intrinsics
 
+    # The MIME type of the sound file
+    attr_accessor :content_type
+
     # Only permits assignments via defined accessors
     def initialize(**attributes)
       attributes.map { |(k, v)| public_send("#{k}=", v) }

data/lib/document.rb

@@ -7,6 +7,7 @@ module FormatParser
     attr_accessor :format
     attr_accessor :document_type
     attr_accessor :page_count
+    attr_accessor :content_type
 
     # Only permits assignments via defined accessors
     def initialize(**attributes)

data/lib/format_parser.rb

@@ -19,6 +19,7 @@ module FormatParser
   require_relative 'io_constraint'
   require_relative 'care'
   require_relative 'active_storage/blob_analyzer'
+  require_relative 'text'
 
   # Define Measurometer in the internal namespace as well
   # so that we stay compatible for the applications that use it
@@ -49,8 +50,10 @@ module FormatParser
     parser_provided_formats = Array(formats)
     parser_provided_natures = Array(natures)
     PARSER_MUX.synchronize do
-      @parsers ||= Set.new
-      @parsers << callable_parser
+      # It can't be a Set because the method `parsers_for` depends on the order
+      # that the parsers were added.
+      @parsers ||= []
+      @parsers << callable_parser unless @parsers.include?(callable_parser)
       @parsers_per_nature ||= {}
       parser_provided_natures.each do |provided_nature|
         @parsers_per_nature[provided_nature] ||= Set.new
@@ -255,7 +258,19 @@ module FormatParser
     # Order the parsers according to their priority value. The ones having a lower
     # value will sort higher and will be applied sooner
     parsers_in_order_of_priority = parsers.to_a.sort do |parser_a, parser_b|
-      @parser_priorities[parser_a] <=> @parser_priorities[parser_b]
+      if @parser_priorities[parser_a] != @parser_priorities[parser_b]
+        @parser_priorities[parser_a] <=> @parser_priorities[parser_b]
+      else
+        # Some parsers have the same priority and we want them to be always sorted
+        # in the same way, to not change the result of FormatParser.parse(results: :first).
+        # When this changes, it can generate flaky tests or event different
+        # results in different environments, which can be hard to understand why.
+        # There is also no guarantee in the order that the elements are added in
+        # @@parser_priorities
+        # So, to have always the same order, we sort by the order that the parsers
+        # were registered if the priorities are the same.
+        @parsers.index(parser_a) <=> @parsers.index(parser_b)
+      end
     end
 
     # If there is one parser that is more likely to match, place it first

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.25.4'
+  VERSION = '0.28.0'
 end

data/lib/image.rb

@@ -64,6 +64,9 @@ module FormatParser
     # it can be placed here
     attr_accessor :intrinsics
 
+    # The MIME type of the image file
+    attr_accessor :content_type
+
     # Only permits assignments via defined accessors
     def initialize(**attributes)
       attributes.map { |(k, v)| public_send("#{k}=", v) }

data/lib/parsers/aiff_parser.rb

@@ -1,6 +1,8 @@
 class FormatParser::AIFFParser
   include FormatParser::IOUtils
 
+  AIFF_MIME_TYPE = 'audio/x-aiff'
+
   # Known chunk types we can omit when parsing,
   # grossly lifted from http://www.muratnkonar.com/aiff/
   KNOWN_CHUNKS = [
@@ -70,7 +72,8 @@ class FormatParser::AIFFParser
       num_audio_channels: channels,
       audio_sample_rate_hz: sample_rate.to_i,
       media_duration_frames: sample_frames,
-      media_duration_seconds: duration_in_seconds
+      media_duration_seconds: duration_in_seconds,
+      content_type: AIFF_MIME_TYPE,
     )
   end
 

data/lib/parsers/bmp_parser.rb

@@ -5,6 +5,7 @@ class FormatParser::BMPParser
 
   VALID_BMP = 'BM'
   PERMISSIBLE_PIXEL_ARRAY_LOCATIONS = 26..512
+  BMP_MIME_TYPE = 'image/bmp'
 
   def likely_match?(filename)
     filename =~ /\.bmp$/i
@@ -42,6 +43,7 @@ class FormatParser::BMPParser
       width_px: width,
       height_px: height,
       color_mode: :rgb,
+      content_type: BMP_MIME_TYPE,
       intrinsics: {
         data_order: data_order,
         bits_per_pixel: bit_depth
@@ -63,6 +65,7 @@ class FormatParser::BMPParser
       width_px: width,
       height_px: height.abs,
       color_mode: :rgb,
+      content_type: BMP_MIME_TYPE,
       intrinsics: {
         vertical_resolution: vertical_res,
         horizontal_resolution: horizontal_res,

data/lib/parsers/cr2_parser.rb

@@ -6,6 +6,7 @@ class FormatParser::CR2Parser
 
   TIFF_HEADER = [0x49, 0x49, 0x2a, 0x00]
   CR2_HEADER  = [0x43, 0x52, 0x02, 0x00]
+  CR2_MIME_TYPE = 'image/x-canon-cr2'
 
   def likely_match?(filename)
     filename =~ /\.cr2$/i
@@ -39,6 +40,7 @@ class FormatParser::CR2Parser
       display_height_px: exif_data.rotated? ? w : h,
       orientation: exif_data.orientation_sym,
       intrinsics: {exif: exif_data},
+      content_type: CR2_MIME_TYPE,
     )
   rescue EXIFR::MalformedTIFF
     nil

data/lib/parsers/dpx_parser.rb

@@ -6,6 +6,11 @@ class FormatParser::DPXParser
   BE_MAGIC = 'SDPX'
   LE_MAGIC = BE_MAGIC.reverse
 
+  # There is no official MIME type for DPX, so we have
+  # to invent something useful. We will prefix it with x-
+  # to indicate that it is a vendor subtype
+  DPX_MIME_TYPE = 'image/x-dpx'
+
   class ByteOrderHintIO < SimpleDelegator
     def initialize(io, is_little_endian)
       super(io)
@@ -35,18 +40,23 @@ class FormatParser::DPXParser
     w = dpx_structure.fetch(:image).fetch(:pixels_per_line)
     h = dpx_structure.fetch(:image).fetch(:lines_per_element)
 
+    display_w = w
+    display_h = h
+
     pixel_aspect_w = dpx_structure.fetch(:orientation).fetch(:horizontal_pixel_aspect)
     pixel_aspect_h = dpx_structure.fetch(:orientation).fetch(:vertical_pixel_aspect)
-    pixel_aspect = pixel_aspect_w / pixel_aspect_h.to_f
 
-    image_aspect = w / h.to_f * pixel_aspect
+    # Find display height and width based on aspect only if the file structure has pixel aspects
+    if pixel_aspect_h != 0 && pixel_aspect_w != 0
+      pixel_aspect = pixel_aspect_w / pixel_aspect_h.to_f
 
-    display_w = w
-    display_h = h
-    if image_aspect > 1
-      display_h = (display_w / image_aspect).round
-    else
-      display_w = (display_h * image_aspect).round
+      image_aspect = w / h.to_f * pixel_aspect
+
+      if image_aspect > 1
+        display_h = (display_w / image_aspect).round
+      else
+        display_w = (display_h * image_aspect).round
+      end
     end
 
     FormatParser::Image.new(
@@ -56,6 +66,7 @@ class FormatParser::DPXParser
       display_width_px: display_w,
       display_height_px: display_h,
       intrinsics: dpx_structure,
+      content_type: DPX_MIME_TYPE,
     )
   end
 

data/lib/parsers/flac_parser.rb

@@ -4,6 +4,7 @@ class FormatParser::FLACParser
   MAGIC_BYTES = 4
   MAGIC_BYTE_STRING = 'fLaC'
   BLOCK_HEADER_BYTES = 4
+  FLAC_MIME_TYPE = 'audio/x-flac'
 
   def likely_match?(filename)
     filename =~ /\.flac$/i
@@ -61,6 +62,7 @@ class FormatParser::FLACParser
       audio_sample_rate_hz: sample_rate,
       media_duration_seconds: duration,
       media_duration_frames: total_samples,
+      content_type: FLAC_MIME_TYPE,
       intrinsics: {
         bits_per_sample: bits_per_sample,
         minimum_frame_size: minimum_frame_size,

data/lib/parsers/gif_parser.rb

@@ -3,6 +3,7 @@ class FormatParser::GIFParser
 
   HEADERS = ['GIF87a', 'GIF89a'].map(&:b)
   NETSCAPE_AND_AUTHENTICATION_CODE = 'NETSCAPE2.0'
+  GIF_MIME_TYPE = 'image/gif'
 
   def likely_match?(filename)
     filename =~ /\.gif$/i
@@ -45,6 +46,7 @@ class FormatParser::GIFParser
       height_px: h,
       has_multiple_frames: is_animated,
       color_mode: :indexed,
+      content_type: GIF_MIME_TYPE
     )
   end
 

data/lib/parsers/jpeg_parser.rb

@@ -12,6 +12,7 @@ class FormatParser::JPEGParser
   APP1_MARKER = 0xE1  # maybe EXIF
   EXIF_MAGIC_STRING = "Exif\0\0".b
   MUST_FIND_NEXT_MARKER_WITHIN_BYTES = 1024
+  JPEG_MIME_TYPE = 'image/jpeg'
 
   def self.likely_match?(filename)
     filename =~ /\.jpe?g$/i
@@ -88,6 +89,7 @@ class FormatParser::JPEGParser
         display_height_px: dh,
         orientation: flat_exif.orientation_sym,
         intrinsics: {exif: flat_exif},
+        content_type: JPEG_MIME_TYPE
       )
 
       return result

data/lib/parsers/m3u_parser.rb

@@ -0,0 +1,23 @@
+class FormatParser::M3UParser
+  include FormatParser::IOUtils
+
+  HEADER = '#EXTM3U'
+  M3U8_MIME_TYPE = 'application/vnd.apple.mpegurl' # https://en.wikipedia.org/wiki/M3U#Internet_media_types
+
+  def likely_match?(filename)
+    filename =~ /\.m3u8?$/i
+  end
+
+  def call(io)
+    io = FormatParser::IOConstraint.new(io)
+
+    header = safe_read(io, 7)
+    return unless HEADER.eql?(header)
+
+    FormatParser::Text.new(
+      format: :m3u,
+      content_type: M3U8_MIME_TYPE,
+    )
+  end
+  FormatParser.register_parser new, natures: :text, formats: :m3u
+end

data/lib/parsers/moov_parser.rb

@@ -11,6 +11,12 @@ class FormatParser::MOOVParser
     'm4a ' => :m4a,
   }
 
+  # https://tools.ietf.org/html/rfc4337#section-2
+  # There is also video/quicktime which we should be able to capture
+  # here, but there is currently no detection for MOVs versus MP4s
+  MP4_AU_MIME_TYPE = 'audio/mp4'
+  MP4_MIXED_MIME_TYPE = 'video/mp4'
+
   def likely_match?(filename)
     filename =~ /\.(mov|m4a|ma4|mp4|aac|m4v)$/i
   end
@@ -49,10 +55,12 @@ class FormatParser::MOOVParser
     end
 
     # M4A only contains audio, while MP4 and friends can contain video.
-    if format_from_moov_type(file_type) == :m4a
+    fmt = format_from_moov_type(file_type)
+    if fmt == :m4a
       FormatParser::Audio.new(
         format: format_from_moov_type(file_type),
         media_duration_seconds: media_duration_s,
+        content_type: MP4_AU_MIME_TYPE,
         intrinsics: atom_tree,
       )
     else
@@ -61,6 +69,7 @@ class FormatParser::MOOVParser
         width_px: width,
         height_px: height,
         media_duration_seconds: media_duration_s,
+        content_type: MP4_MIXED_MIME_TYPE,
         intrinsics: atom_tree,
       )
     end