format_parser 0.25.5 → 0.29.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c39f9a0aae6e1474bbe2e3ff845ebf6c349e042510a4e8049384bfcdb7506c33
-  data.tar.gz: 604732d31301b227844649b82b3c4e13aa5dddcab1158cfea51abc7998ebd26e
+  metadata.gz: b8efef8742cfefdcde0aa05ab32c726004325c2210b19e25ca2cf866ad786074
+  data.tar.gz: bbd59f6f046a35cff93ea4e20a5168c2912007b06b3fabe7a5b23f7a0ed70c34
 SHA512:
-  metadata.gz: 189c3e155e868475ce45cbd4576f2d381452b74a1718962f9a306915837d415ccb699b5e8bb0c84b282013890a6a1b4a4e873227525154e577b39e9c6e7f31d9
-  data.tar.gz: c300a554bb9c2328abafe79ca560de504b6684e96a4a97b698bc8c555085bd0a423792965cb1d323306108be47bf7e238af3bc0e44bb858c0af5aaef9d45546e
+  metadata.gz: b4a7d965be6ff0cb0abf8cdec5f4d31259aada155ae465c1b31d5eba718f9d0071ee8b7a6b2061e6024add45d32b901c0f95a37373727a59fd4a6562041f52e9
+  data.tar.gz: d314cb6876185ddedcef01c67205cf0635c2d8aebd0871dfd7834f0f54c15c38c31db7cc730c6aea8f8a72aff85949fa6937d970dfb320120a5b8c843f003753

data/.github/workflows/main.yml

@@ -0,0 +1,104 @@
+name: CI
+
+on: [push,pull_request]
+
+env:
+  BUNDLE_PATH: vendor/bundle
+
+jobs:
+  lint:
+    name: Code Style
+    runs-on: ubuntu-18.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    strategy:
+      matrix:
+        ruby:
+          - 2.7
+          - 2.6
+          - 2.5
+          - 2.4
+          - 2.3
+          - 2.2
+          - jruby
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Gemfile Cache
+        uses: actions/cache@v2
+        with:
+          path: Gemfile.lock
+          key: ${{ runner.os }}-gemlock-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'format_parser.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gemlock-${{ matrix.ruby }}-
+      - name: Bundle Cache
+        id: cache-gems
+        uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'Gemfile.lock', 'format_parser.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-${{ matrix.ruby }}-
+            ${{ runner.os }}-gems-
+      - name: Bundle Install
+        if: steps.cache-gems.outputs.cache-hit != 'true'
+        run: bundle install --jobs 4 --retry 3
+      - name: Rubocop Cache
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/rubocop_cache
+          key: ${{ runner.os }}-rubocop-${{ hashFiles('.rubocop.yml') }}
+          restore-keys: |
+            ${{ runner.os }}-rubocop-
+      - name: Rubocop
+        run: bundle exec rubocop
+  test:
+    name: Specs
+    runs-on: ubuntu-18.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    strategy:
+      matrix:
+        ruby:
+          - 2.7
+          - 2.6
+          - 2.5
+          - 2.4
+          - 2.3
+          - 2.2
+          - jruby
+        experimental: [false]
+        include:
+          - ruby: 3.0
+            experimental: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Gemfile Cache
+        uses: actions/cache@v2
+        with:
+          path: Gemfile.lock
+          key: ${{ runner.os }}-gemlock-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'format_parser.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gemlock-${{ matrix.ruby }}-
+      - name: Bundle Cache
+        id: cache-gems
+        uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'Gemfile.lock', 'format_parser.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-${{ matrix.ruby }}-
+            ${{ runner.os }}-gems-
+      - name: Bundle Install
+        if: steps.cache-gems.outputs.cache-hit != 'true'
+        run: bundle install --jobs 4 --retry 3
+      - name: RSpec
+        continue-on-error: ${{ matrix.experimental }}
+        run: bundle exec rake parallel:spec

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 0.29.0
+* Add option `headers:` to `FormatParser.parse_http`
+
+## 0.28.0
+* Change `FormatParser.parse_http` to follow HTTP redirects
+
+## 0.27.0
+* Add `#content_type` on `Result` return values which makes sense for the detected filetype
+
+## 0.26.0
+* Add support for M3U format files
+
+## 0.25.6
+* Fix FormatParser.parse (with `results: :first`) to be deterministic
+
 ## 0.25.5
 * DPX: Fix DPXParser to support images without aspect ratio
 

data/README.md

@@ -32,6 +32,7 @@ and [dimensions,](https://github.com/sstephenson/dimensions) borrowing from them
 * DOCX, PPTX, XLSX
 * OGG
 * MPEG, MPG
+* M3U
 
 ...with [more](https://github.com/WeTransfer/format_parser/issues?q=is%3Aissue+is%3Aopen+label%3Aformats) on the way!
 
@@ -194,6 +195,9 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
   manipulated using the [https://github.com/recurser/exif-orientation-examples](exif-orientation-examples)
   script.
 
+### M3U
+- The M3U fixture files were created by one of the project maintainers
+
 ### .key
 - The `keynote_recognized_as_jpeg.key` file was created by the project maintainers
 

data/format_parser.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'exifr', '~> 1', '>= 1.3.8'
   spec.add_dependency 'id3tag', '~> 0.14'
   spec.add_dependency 'faraday', '~> 0.13'
+  spec.add_dependency 'faraday_middleware', '~> 0.14'
   spec.add_dependency 'measurometer', '~> 1'
 
   spec.add_development_dependency 'rspec', '~> 3.0'

data/lib/archive.rb

@@ -26,6 +26,9 @@ module FormatParser
     # it can be placed here
     attr_accessor :intrinsics
 
+    # The MIME type of the archive
+    attr_accessor :content_type
+
     # Only permits assignments via defined accessors
     def initialize(**attributes)
       attributes.map { |(k, v)| public_send("#{k}=", v) }

data/lib/audio.rb

@@ -35,6 +35,9 @@ module FormatParser
     # it can be placed here
     attr_accessor :intrinsics
 
+    # The MIME type of the sound file
+    attr_accessor :content_type
+
     # Only permits assignments via defined accessors
     def initialize(**attributes)
       attributes.map { |(k, v)| public_send("#{k}=", v) }

data/lib/document.rb

@@ -7,6 +7,7 @@ module FormatParser
     attr_accessor :format
     attr_accessor :document_type
     attr_accessor :page_count
+    attr_accessor :content_type
 
     # Only permits assignments via defined accessors
     def initialize(**attributes)

data/lib/format_parser.rb

@@ -19,6 +19,7 @@ module FormatParser
   require_relative 'io_constraint'
   require_relative 'care'
   require_relative 'active_storage/blob_analyzer'
+  require_relative 'text'
 
   # Define Measurometer in the internal namespace as well
   # so that we stay compatible for the applications that use it
@@ -49,8 +50,10 @@ module FormatParser
     parser_provided_formats = Array(formats)
     parser_provided_natures = Array(natures)
     PARSER_MUX.synchronize do
-      @parsers ||= Set.new
-      @parsers << callable_parser
+      # It can't be a Set because the method `parsers_for` depends on the order
+      # that the parsers were added.
+      @parsers ||= []
+      @parsers << callable_parser unless @parsers.include?(callable_parser)
       @parsers_per_nature ||= {}
       parser_provided_natures.each do |provided_nature|
         @parsers_per_nature[provided_nature] ||= Set.new
@@ -85,13 +88,14 @@ module FormatParser
   # given to `.parse`. The accepted keyword arguments are the same as the ones for `parse`.
   #
   # @param url[String, URI] the HTTP(S) URL to request the object from using Faraday and `Range:` requests
+  # @param headers[Hash] (optional) the HTTP headers to request the object from using Faraday
   # @param kwargs the keyword arguments to be delegated to `.parse`
   # @see {.parse}
-  def self.parse_http(url, **kwargs)
+  def self.parse_http(url, headers: {}, **kwargs)
     # Do not extract the filename, since the URL
     # can really be "anything". But if the caller
     # provides filename_hint it will be carried over
-    parse(RemoteIO.new(url), **kwargs)
+    parse(RemoteIO.new(url, headers: headers), **kwargs)
   end
 
   # Parses the file at the given `path` and returns the results as if it were any IO
@@ -255,7 +259,19 @@ module FormatParser
     # Order the parsers according to their priority value. The ones having a lower
     # value will sort higher and will be applied sooner
     parsers_in_order_of_priority = parsers.to_a.sort do |parser_a, parser_b|
-      @parser_priorities[parser_a] <=> @parser_priorities[parser_b]
+      if @parser_priorities[parser_a] != @parser_priorities[parser_b]
+        @parser_priorities[parser_a] <=> @parser_priorities[parser_b]
+      else
+        # Some parsers have the same priority and we want them to be always sorted
+        # in the same way, to not change the result of FormatParser.parse(results: :first).
+        # When this changes, it can generate flaky tests or event different
+        # results in different environments, which can be hard to understand why.
+        # There is also no guarantee in the order that the elements are added in
+        # @@parser_priorities
+        # So, to have always the same order, we sort by the order that the parsers
+        # were registered if the priorities are the same.
+        @parsers.index(parser_a) <=> @parsers.index(parser_b)
+      end
     end
 
     # If there is one parser that is more likely to match, place it first

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.25.5'
+  VERSION = '0.29.0'
 end

data/lib/image.rb

@@ -64,6 +64,9 @@ module FormatParser
     # it can be placed here
     attr_accessor :intrinsics
 
+    # The MIME type of the image file
+    attr_accessor :content_type
+
     # Only permits assignments via defined accessors
     def initialize(**attributes)
       attributes.map { |(k, v)| public_send("#{k}=", v) }

data/lib/parsers/aiff_parser.rb

@@ -1,6 +1,8 @@
 class FormatParser::AIFFParser
   include FormatParser::IOUtils
 
+  AIFF_MIME_TYPE = 'audio/x-aiff'
+
   # Known chunk types we can omit when parsing,
   # grossly lifted from http://www.muratnkonar.com/aiff/
   KNOWN_CHUNKS = [
@@ -70,7 +72,8 @@ class FormatParser::AIFFParser
       num_audio_channels: channels,
       audio_sample_rate_hz: sample_rate.to_i,
       media_duration_frames: sample_frames,
-      media_duration_seconds: duration_in_seconds
+      media_duration_seconds: duration_in_seconds,
+      content_type: AIFF_MIME_TYPE,
     )
   end
 

data/lib/parsers/bmp_parser.rb

@@ -5,6 +5,7 @@ class FormatParser::BMPParser
 
   VALID_BMP = 'BM'
   PERMISSIBLE_PIXEL_ARRAY_LOCATIONS = 26..512
+  BMP_MIME_TYPE = 'image/bmp'
 
   def likely_match?(filename)
     filename =~ /\.bmp$/i
@@ -42,6 +43,7 @@ class FormatParser::BMPParser
       width_px: width,
       height_px: height,
       color_mode: :rgb,
+      content_type: BMP_MIME_TYPE,
       intrinsics: {
         data_order: data_order,
         bits_per_pixel: bit_depth
@@ -63,6 +65,7 @@ class FormatParser::BMPParser
       width_px: width,
       height_px: height.abs,
       color_mode: :rgb,
+      content_type: BMP_MIME_TYPE,
       intrinsics: {
         vertical_resolution: vertical_res,
         horizontal_resolution: horizontal_res,

data/lib/parsers/cr2_parser.rb

@@ -6,6 +6,7 @@ class FormatParser::CR2Parser
 
   TIFF_HEADER = [0x49, 0x49, 0x2a, 0x00]
   CR2_HEADER  = [0x43, 0x52, 0x02, 0x00]
+  CR2_MIME_TYPE = 'image/x-canon-cr2'
 
   def likely_match?(filename)
     filename =~ /\.cr2$/i
@@ -39,6 +40,7 @@ class FormatParser::CR2Parser
       display_height_px: exif_data.rotated? ? w : h,
       orientation: exif_data.orientation_sym,
       intrinsics: {exif: exif_data},
+      content_type: CR2_MIME_TYPE,
     )
   rescue EXIFR::MalformedTIFF
     nil

data/lib/parsers/dpx_parser.rb

@@ -6,6 +6,11 @@ class FormatParser::DPXParser
   BE_MAGIC = 'SDPX'
   LE_MAGIC = BE_MAGIC.reverse
 
+  # There is no official MIME type for DPX, so we have
+  # to invent something useful. We will prefix it with x-
+  # to indicate that it is a vendor subtype
+  DPX_MIME_TYPE = 'image/x-dpx'
+
   class ByteOrderHintIO < SimpleDelegator
     def initialize(io, is_little_endian)
       super(io)
@@ -61,6 +66,7 @@ class FormatParser::DPXParser
       display_width_px: display_w,
       display_height_px: display_h,
       intrinsics: dpx_structure,
+      content_type: DPX_MIME_TYPE,
     )
   end
 

data/lib/parsers/flac_parser.rb

@@ -4,6 +4,7 @@ class FormatParser::FLACParser
   MAGIC_BYTES = 4
   MAGIC_BYTE_STRING = 'fLaC'
   BLOCK_HEADER_BYTES = 4
+  FLAC_MIME_TYPE = 'audio/x-flac'
 
   def likely_match?(filename)
     filename =~ /\.flac$/i
@@ -61,6 +62,7 @@ class FormatParser::FLACParser
       audio_sample_rate_hz: sample_rate,
       media_duration_seconds: duration,
       media_duration_frames: total_samples,
+      content_type: FLAC_MIME_TYPE,
       intrinsics: {
         bits_per_sample: bits_per_sample,
         minimum_frame_size: minimum_frame_size,

data/lib/parsers/gif_parser.rb

@@ -3,6 +3,7 @@ class FormatParser::GIFParser
 
   HEADERS = ['GIF87a', 'GIF89a'].map(&:b)
   NETSCAPE_AND_AUTHENTICATION_CODE = 'NETSCAPE2.0'
+  GIF_MIME_TYPE = 'image/gif'
 
   def likely_match?(filename)
     filename =~ /\.gif$/i
@@ -45,6 +46,7 @@ class FormatParser::GIFParser
       height_px: h,
       has_multiple_frames: is_animated,
       color_mode: :indexed,
+      content_type: GIF_MIME_TYPE
     )
   end
 

data/lib/parsers/jpeg_parser.rb

@@ -12,6 +12,7 @@ class FormatParser::JPEGParser
   APP1_MARKER = 0xE1  # maybe EXIF
   EXIF_MAGIC_STRING = "Exif\0\0".b
   MUST_FIND_NEXT_MARKER_WITHIN_BYTES = 1024
+  JPEG_MIME_TYPE = 'image/jpeg'
 
   def self.likely_match?(filename)
     filename =~ /\.jpe?g$/i
@@ -88,6 +89,7 @@ class FormatParser::JPEGParser
         display_height_px: dh,
         orientation: flat_exif.orientation_sym,
         intrinsics: {exif: flat_exif},
+        content_type: JPEG_MIME_TYPE
       )
 
       return result

data/lib/parsers/m3u_parser.rb

@@ -0,0 +1,23 @@
+class FormatParser::M3UParser
+  include FormatParser::IOUtils
+
+  HEADER = '#EXTM3U'
+  M3U8_MIME_TYPE = 'application/vnd.apple.mpegurl' # https://en.wikipedia.org/wiki/M3U#Internet_media_types
+
+  def likely_match?(filename)
+    filename =~ /\.m3u8?$/i
+  end
+
+  def call(io)
+    io = FormatParser::IOConstraint.new(io)
+
+    header = safe_read(io, 7)
+    return unless HEADER.eql?(header)
+
+    FormatParser::Text.new(
+      format: :m3u,
+      content_type: M3U8_MIME_TYPE,
+    )
+  end
+  FormatParser.register_parser new, natures: :text, formats: :m3u
+end

data/lib/parsers/moov_parser.rb

@@ -11,6 +11,12 @@ class FormatParser::MOOVParser
     'm4a ' => :m4a,
   }
 
+  # https://tools.ietf.org/html/rfc4337#section-2
+  # There is also video/quicktime which we should be able to capture
+  # here, but there is currently no detection for MOVs versus MP4s
+  MP4_AU_MIME_TYPE = 'audio/mp4'
+  MP4_MIXED_MIME_TYPE = 'video/mp4'
+
   def likely_match?(filename)
     filename =~ /\.(mov|m4a|ma4|mp4|aac|m4v)$/i
   end
@@ -49,10 +55,12 @@ class FormatParser::MOOVParser
     end
 
     # M4A only contains audio, while MP4 and friends can contain video.
-    if format_from_moov_type(file_type) == :m4a
+    fmt = format_from_moov_type(file_type)
+    if fmt == :m4a
       FormatParser::Audio.new(
         format: format_from_moov_type(file_type),
         media_duration_seconds: media_duration_s,
+        content_type: MP4_AU_MIME_TYPE,
         intrinsics: atom_tree,
       )
     else
@@ -61,6 +69,7 @@ class FormatParser::MOOVParser
         width_px: width,
         height_px: height,
         media_duration_seconds: media_duration_s,
+        content_type: MP4_MIXED_MIME_TYPE,
         intrinsics: atom_tree,
       )
     end