format_parser 0.25.4 → 0.28.0

data/lib/parsers/mp3_parser.rb

@@ -32,7 +32,7 @@ class FormatParser::MP3Parser
   MAGIC_LE = [0x49, 0x49, 0x2A, 0x0].pack('C4')
   MAGIC_BE = [0x4D, 0x4D, 0x0, 0x2A].pack('C4')
   TIFF_HEADER_BYTES = [MAGIC_LE, MAGIC_BE]
-
+  MP3_MIME_TYPE = 'audio/mpeg'
   # Wraps the Tag object returned by ID3Tag in such
   # a way that a usable JSON representation gets
   # returned
@@ -104,7 +104,8 @@ class FormatParser::MP3Parser
       # do not tell anything of substance
       num_audio_channels: first_frame.channels,
       audio_sample_rate_hz: first_frame.sample_rate,
-      intrinsics: id3tags_hash.merge(id3tags: tags)
+      intrinsics: id3tags_hash.merge(id3tags: tags),
+      content_type: MP3_MIME_TYPE,
     )
 
     extra_file_attirbutes = fetch_extra_attributes_from_id3_tags(id3tags_hash)

data/lib/parsers/ogg_parser.rb

@@ -3,8 +3,8 @@
 class FormatParser::OggParser
   include FormatParser::IOUtils
 
-  # Maximum size of an Ogg page
   MAX_POSSIBLE_PAGE_SIZE = 65307
+  OGG_MIME_TYPE = 'audio/ogg'
 
   def likely_match?(filename)
     filename =~ /\.ogg$/i
@@ -45,7 +45,8 @@ class FormatParser::OggParser
       format: :ogg,
       audio_sample_rate_hz: sample_rate,
       num_audio_channels: channels,
-      media_duration_seconds: duration
+      media_duration_seconds: duration,
+      content_type: OGG_MIME_TYPE,
     )
   end
 

data/lib/parsers/pdf_parser.rb

@@ -1,6 +1,5 @@
 class FormatParser::PDFParser
   include FormatParser::IOUtils
-
   # First 9 bytes of a PDF should be in this format, according to:
   #
   #  https://stackoverflow.com/questions/3108201/detect-if-pdf-file-is-correct-header-pdf
@@ -8,6 +7,7 @@ class FormatParser::PDFParser
   # There are however exceptions, which are left out for now.
   #
   PDF_MARKER = /%PDF-1\.[0-8]{1}/
+  PDF_CONTENT_TYPE = 'application/pdf'
 
   def likely_match?(filename)
     filename =~ /\.(pdf|ai)$/i
@@ -18,7 +18,7 @@ class FormatParser::PDFParser
 
     return unless safe_read(io, 9) =~ PDF_MARKER
 
-    FormatParser::Document.new(format: :pdf)
+    FormatParser::Document.new(format: :pdf, content_type: PDF_CONTENT_TYPE)
   end
 
   FormatParser.register_parser new, natures: :document, formats: :pdf, priority: 1

data/lib/parsers/png_parser.rb

@@ -14,6 +14,7 @@ class FormatParser::PNGParser
     4 => true, # Grayscale with alpha
     6 => true,
   }
+  PNG_MIME_TYPE = 'image/png'
 
   def likely_match?(filename)
     filename =~ /\.png$/i
@@ -67,6 +68,7 @@ class FormatParser::PNGParser
       color_mode: color_mode,
       has_multiple_frames: has_animation,
       num_animation_or_video_frames: num_frames,
+      content_type: PNG_MIME_TYPE,
     )
   end
 

data/lib/parsers/psd_parser.rb

@@ -2,6 +2,7 @@ class FormatParser::PSDParser
   include FormatParser::IOUtils
 
   PSD_HEADER = [0x38, 0x42, 0x50, 0x53]
+  PSD_MIME_TYPE = 'application/x-photoshop'
 
   def likely_match?(filename)
     filename =~ /\.psd$/i # Maybe also PSB at some point
@@ -20,6 +21,7 @@ class FormatParser::PSDParser
       format: :psd,
       width_px: w,
       height_px: h,
+      content_type: PSD_MIME_TYPE,
     )
   end
 

data/lib/parsers/tiff_parser.rb

@@ -5,6 +5,8 @@ class FormatParser::TIFFParser
   MAGIC_LE = [0x49, 0x49, 0x2A, 0x0].pack('C4')
   MAGIC_BE = [0x4D, 0x4D, 0x0, 0x2A].pack('C4')
   HEADER_BYTES = [MAGIC_LE, MAGIC_BE]
+  TIFF_MIME_TYPE = 'image/tiff'
+  ARW_MIME_TYPE = 'image/x-sony-arw'
 
   def likely_match?(filename)
     filename =~ /\.tiff?$/i
@@ -14,7 +16,10 @@ class FormatParser::TIFFParser
     io = FormatParser::IOConstraint.new(io)
 
     return unless HEADER_BYTES.include?(safe_read(io, 4))
-    io.seek(io.pos + 2) # Skip over the offset of the IFD, EXIFR will re-read it anyway
+
+    # Skip over the offset of the IFD,
+    # EXIFR will re-read it anyway
+    io.seek(io.pos + 2)
     return if cr2?(io)
 
     # The TIFF scanner in EXIFR is plenty good enough,
@@ -26,14 +31,17 @@ class FormatParser::TIFFParser
     w = exif_data.width || exif_data.pixel_x_dimension
     h = exif_data.height || exif_data.pixel_y_dimension
 
+    format = arw?(exif_data) ? :arw : :tif
+    mime_type = arw?(exif_data) ? ARW_MIME_TYPE : TIFF_MIME_TYPE
     FormatParser::Image.new(
-      format: arw?(exif_data) ? :arw : :tif, # Specify format as arw for Sony ARW format images, else tif
+      format: format,
       width_px: w,
       height_px: h,
       display_width_px: exif_data.rotated? ? h : w,
       display_height_px: exif_data.rotated? ? w : h,
       orientation: exif_data.orientation_sym,
       intrinsics: {exif: exif_data},
+      content_type: mime_type,
     )
   rescue EXIFR::MalformedTIFF
     nil

data/lib/parsers/wav_parser.rb

@@ -1,6 +1,8 @@
 class FormatParser::WAVParser
   include FormatParser::IOUtils
 
+  WAV_MIME_TYPE = 'audio/x-wav'
+
   def likely_match?(filename)
     filename =~ /\.wav$/i
   end
@@ -96,6 +98,7 @@ class FormatParser::WAVParser
       audio_sample_rate_hz: fmt_data[:sample_rate],
       media_duration_frames: sample_frames,
       media_duration_seconds: duration_in_seconds,
+      content_type: WAV_MIME_TYPE,
     )
   end
 

data/lib/parsers/zip_parser.rb

@@ -5,6 +5,8 @@ class FormatParser::ZIPParser
   include OfficeFormats
   include FormatParser::IOUtils
 
+  ZIP_MIME_TYPE = 'application/zip'
+
   def likely_match?(filename)
     filename =~ /\.(zip|docx|keynote|numbers|pptx|xlsx)$/i
   end
@@ -25,10 +27,10 @@ class FormatParser::ZIPParser
     end
 
     if office_document?(filenames_set)
-      office_format = office_file_format_from_entry_set(filenames_set)
-      FormatParser::Archive.new(nature: :document, format: office_format, entries: entries_archive)
+      office_format, mime_type = office_file_format_and_mime_type_from_entry_set(filenames_set)
+      FormatParser::Archive.new(nature: :document, format: office_format, entries: entries_archive, content_type: mime_type)
     else
-      FormatParser::Archive.new(nature: :archive,  format: :zip, entries: entries_archive)
+      FormatParser::Archive.new(nature: :archive,  format: :zip, entries: entries_archive, content_type: ZIP_MIME_TYPE)
     end
   rescue FileReader::Error
     # This is not a ZIP, or a broken ZIP.

data/lib/parsers/zip_parser/office_formats.rb

@@ -37,15 +37,15 @@ module FormatParser::ZIPParser::OfficeFormats
     OFFICE_MARKER_FILES.subset?(filenames_set)
   end
 
-  def office_file_format_from_entry_set(filenames_set)
+  def office_file_format_and_mime_type_from_entry_set(filenames_set)
     if filenames_set.include?('word/document.xml')
-      :docx
+      [:docx, 'application/vnd.openxmlformats-officedocument.wordprocessingml.document']
     elsif filenames_set.include?('xl/workbook.xml')
-      :xlsx
+      [:xlsx, 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']
     elsif filenames_set.include?('ppt/presentation.xml')
-      :pptx
+      [:pptx, 'application/vnd.openxmlformats-officedocument.presentationml.presentation']
     else
-      :unknown
+      [:unknown, 'application/zip']
     end
   end
 end

data/lib/remote_io.rb

@@ -26,6 +26,7 @@ class FormatParser::RemoteIO
   # @param uri[URI, String] the remote URL to obtain
   def initialize(uri)
     require 'faraday'
+    require 'faraday_middleware/response/follow_redirects'
     @uri = uri
     @pos = 0
     @remote_size = false
@@ -78,7 +79,12 @@ class FormatParser::RemoteIO
     # We use a GET and not a HEAD request followed by a GET because
     # S3 does not allow HEAD requests if you only presigned your URL for GETs, so we
     # combine the first GET of a segment and retrieving the size of the resource
-    response = Faraday.get(@uri, nil, range: 'bytes=%d-%d' % [range.begin, range.end])
+    conn = Faraday.new do |faraday|
+      faraday.use FaradayMiddleware::FollowRedirects
+      # we still need the default adapter, more details: https://blog.thecodewhisperer.com/permalink/losing-time-to-faraday
+      faraday.adapter Faraday.default_adapter
+    end
+    response = conn.get(@uri, nil, range: 'bytes=%d-%d' % [range.begin, range.end])
 
     case response.status
     when 200, 206

data/lib/text.rb

@@ -0,0 +1,19 @@
+module FormatParser
+  class Text
+    include FormatParser::AttributesJSON
+
+    NATURE = :text
+
+    attr_accessor :format
+    attr_accessor :content_type
+
+    # Only permits assignments via defined accessors
+    def initialize(**attributes)
+      attributes.map { |(k, v)| public_send("#{k}=", v) }
+    end
+
+    def nature
+      NATURE
+    end
+  end
+end

data/lib/video.rb

@@ -23,6 +23,9 @@ module FormatParser
     # it can be placed here
     attr_accessor :intrinsics
 
+    # The MIME type of the video
+    attr_accessor :content_type
+
     # Only permits assignments via defined accessors
     def initialize(**attributes)
       attributes.map { |(k, v)| public_send("#{k}=", v) }

data/spec/format_parser_spec.rb

@@ -173,6 +173,26 @@ describe FormatParser do
       prioritized_parsers = FormatParser.parsers_for([:archive, :document, :image, :audio], [:tif, :jpg, :zip, :docx, :mp3, :aiff], 'a-file.zip')
       expect(prioritized_parsers.first).to be_kind_of(FormatParser::ZIPParser)
     end
+
+    it 'sorts the parsers by priority and name' do
+      parsers = FormatParser.parsers_for(
+        [:audio, :image],
+        [:cr2, :dpx, :fdx, :flac, :gif, :jpg, :mov, :mp4, :m4a, :mp3, :mpg, :mpeg, :ogg, :png, :tif, :wav]
+      )
+
+      expect(parsers.map { |parser| parser.class.name }).to eq([
+        'FormatParser::GIFParser',
+        'Class',
+        'FormatParser::PNGParser',
+        'FormatParser::CR2Parser',
+        'FormatParser::DPXParser',
+        'FormatParser::FLACParser',
+        'FormatParser::MP3Parser',
+        'FormatParser::OggParser',
+        'FormatParser::TIFFParser',
+        'FormatParser::WAVParser'
+      ])
+    end
   end
 
   describe '.register_parser and .deregister_parser' do

data/spec/parsers/aiff_parser_spec.rb

@@ -10,6 +10,7 @@ describe FormatParser::AIFFParser do
     expect(parse_result.num_audio_channels).to eq(2)
     expect(parse_result.audio_sample_rate_hz).to be_within(0.01).of(44100)
     expect(parse_result.media_duration_seconds).to be_within(0.01).of(1.05)
+    expect(parse_result.content_type).to eq('audio/x-aiff')
   end
 
   it 'parses a Logic Pro created AIFF sample file having a COMT chunk before a COMM chunk' do

data/spec/parsers/bmp_parser_spec.rb

@@ -13,6 +13,8 @@ describe FormatParser::BMPParser do
     expect(parsed.width_px).to eq(40)
     expect(parsed.height_px).to eq(27)
 
+    expect(parsed.content_type).to eq('image/bmp')
+
     expect(parsed.intrinsics).not_to be_nil
     expect(parsed.intrinsics[:vertical_resolution]).to eq(2834)
     expect(parsed.intrinsics[:horizontal_resolution]).to eq(2834)
@@ -32,6 +34,8 @@ describe FormatParser::BMPParser do
     expect(parsed.width_px).to eq(1920)
     expect(parsed.height_px).to eq(1080)
 
+    expect(parsed.content_type).to eq('image/bmp')
+
     expect(parsed.intrinsics).not_to be_nil
     expect(parsed.intrinsics[:vertical_resolution]).to eq(2835)
     expect(parsed.intrinsics[:horizontal_resolution]).to eq(2835)
@@ -51,6 +55,8 @@ describe FormatParser::BMPParser do
     expect(parsed.width_px).to eq(200)
     expect(parsed.height_px).to eq(200)
 
+    expect(parsed.content_type).to eq('image/bmp')
+
     expect(parsed.intrinsics).not_to be_nil
   end
 
@@ -64,6 +70,7 @@ describe FormatParser::BMPParser do
     expect(parsed.color_mode).to eq(:rgb)
     expect(parsed.width_px).to eq(40)
     expect(parsed.height_px).to eq(27)
+    expect(parsed.content_type).to eq('image/bmp')
     expect(parsed.intrinsics[:bits_per_pixel]).to eq(24)
     expect(parsed.intrinsics[:data_order]).to eq(:normal)
 
@@ -76,6 +83,7 @@ describe FormatParser::BMPParser do
     expect(parsed.color_mode).to eq(:rgb)
     expect(parsed.width_px).to eq(40)
     expect(parsed.height_px).to eq(27)
+    expect(parsed.content_type).to eq('image/bmp')
     expect(parsed.intrinsics[:bits_per_pixel]).to eq(24)
     expect(parsed.intrinsics[:data_order]).to eq(:normal)
   end

data/spec/parsers/cr2_parser_spec.rb

@@ -17,6 +17,7 @@ describe FormatParser::CR2Parser do
         expect(parsed.height_px).to be > 0
 
         expect(parsed.orientation).not_to be_nil
+        expect(parsed.content_type).to eq('image/x-canon-cr2')
       end
     end
   end

data/spec/parsers/dpx_parser_spec.rb

@@ -15,6 +15,7 @@ describe FormatParser::DPXParser do
       expect(parsed.width_px).to be_between(0, 2048)
       expect(parsed.height_px).to be_kind_of(Integer)
       expect(parsed.height_px).to be_between(0, 4000)
+      expect(parsed.content_type).to eq('image/x-dpx')
     end
   end
 

data/spec/parsers/flac_parser_spec.rb

@@ -14,6 +14,7 @@ describe FormatParser::FLACParser do
     expect(parsed.intrinsics).not_to be_nil
     expect(parsed.media_duration_frames).to eq(33810)
     expect(parsed.media_duration_seconds).to be_within(0.1).of(0.836)
+    expect(parsed.content_type).to eq('audio/x-flac')
   end
 
   it 'decodes and estimates duration for the 16bit FLAC File' do

data/spec/parsers/gif_parser_spec.rb

@@ -17,6 +17,7 @@ describe FormatParser::GIFParser do
 
         expect(parsed.height_px).to be_kind_of(Integer)
         expect(parsed.height_px).to be > 0
+        expect(parsed.content_type).to eq('image/gif')
       end
     end
   end

data/spec/parsers/jpeg_parser_spec.rb

@@ -14,6 +14,7 @@ describe FormatParser::JPEGParser do
 
         expect(parsed.height_px).to be_kind_of(Integer)
         expect(parsed.height_px).to be > 0
+        expect(parsed.content_type).to eq('image/jpeg')
       end
     end
   end

data/spec/parsers/m3u_parser_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+describe FormatParser::M3UParser do
+  let(:parsed_m3u) do
+    subject.call(
+      File.open(
+        Pathname.new(fixtures_dir).join('M3U').join(m3u_file),
+        'rb'
+      )
+    )
+  end
+
+  describe 'an m3u file with missing header' do
+    let(:m3u_file) { 'plain_text.m3u' }
+
+    it 'does not parse the file successfully' do
+      expect(parsed_m3u).to be_nil
+    end
+  end
+
+  describe 'an m3u file with valid header' do
+    let(:m3u_file) { 'sample.m3u' }
+
+    it 'parses the file successfully' do
+      expect(parsed_m3u).not_to be_nil
+      expect(parsed_m3u.nature).to eq(:text)
+      expect(parsed_m3u.format).to eq(:m3u)
+      expect(parsed_m3u.content_type).to eq('application/vnd.apple.mpegurl')
+    end
+  end
+
+  describe 'an m3u8 file with valid header' do
+    let(:m3u_file) { 'sample.m3u8' }
+
+    it 'parses the file successfully' do
+      expect(parsed_m3u).not_to be_nil
+      expect(parsed_m3u.nature).to eq(:text)
+      expect(parsed_m3u.format).to eq(:m3u)
+    end
+  end
+end

data/spec/parsers/moov_parser_spec.rb

@@ -37,7 +37,7 @@ describe FormatParser::MOOVParser do
       expect(result.nature).to eq(:audio)
       expect(result.media_duration_seconds).to be_kind_of(Float)
       expect(result.media_duration_seconds).to be > 0
-
+      expect(result.content_type).to be_kind_of(String)
       expect(result.intrinsics).not_to be_nil
     end
   end
@@ -52,6 +52,7 @@ describe FormatParser::MOOVParser do
       expect(result.height_px).to be > 0
       expect(result.media_duration_seconds).to be_kind_of(Float)
       expect(result.media_duration_seconds).to be > 0
+      expect(result.content_type).to eq('video/mp4')
 
       expect(result.intrinsics).not_to be_nil
     end
@@ -67,6 +68,7 @@ describe FormatParser::MOOVParser do
       expect(result.height_px).to be > 0
       expect(result.media_duration_seconds).to be_kind_of(Float)
       expect(result.media_duration_seconds).to be > 0
+      expect(result.content_type).to eq('video/mp4')
 
       expect(result.intrinsics).not_to be_nil
     end
@@ -79,6 +81,7 @@ describe FormatParser::MOOVParser do
     expect(result).not_to be_nil
     expect(result.nature).to eq(:audio)
     expect(result.format).to eq(:m4a)
+    expect(result.content_type).to eq('audio/mp4')
   end
 
   it 'parses a MOV file and provides the necessary metadata' do