format_parser 0.1.7 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 70fd9b84e2b397e862c2f6554eb6f3830f77b3c8
-  data.tar.gz: 36c2a28a96f0bea5644f550bb616983d8320961d
+  metadata.gz: aac884390c22699e4c3508ef5ecd363d2f1d2b5a
+  data.tar.gz: da10c68c25b0ba46929282a1bcd918e9bf4513b1
 SHA512:
-  metadata.gz: d2603890e62f99e5a4e63dd28f1860aeee86f1f47d0a690fa559f01a27aa106104fb71195427814ac346208435a3fd720db4458cdbd8cb9cf1291c517334ca8f
-  data.tar.gz: 35f16d20d4a3e015d58b82297f66f2bd3df1e2ef78a0ced278c474bcec64e8d47241fc4e99368b1fa8d203bda182e509c0c58d65037539f9e7b6d5f7b98bfe0e
+  metadata.gz: 74efa5fe85532815e9475e711af8cfc6a2a4478c1832479ec073f7fdecdf4ed2997a61d2c5e068fe6013cd8fb9fbea7c16eaf1bc109485dde7d469b4cec62342
+  data.tar.gz: 2a8da42f8d4f49dff0b7c76393d5c9385ef109ada91d539235d5afbef98689e5f99ca4a5b9a99843c109d2665ea1c4843192d3cf9c25d4ccb3117c98c25fb455

data/README.md

@@ -15,18 +15,57 @@ and [dimensions,](https://github.com/sstephenson/dimensions) borrowing from them
 
 ## Basic usage
 
-Pass an IO object that responds to `read` and `seek` to `FormatParser`.
+Pass an IO object that responds to `read` and `seek` to `FormatParser` and an array of matches will be returned.
 
 ```ruby
-file_info = FormatParser.parse(File.open("myimage.jpg", "rb"))
-file_info.file_nature           #=> :image
-file_info.file_format           #=> :JPG
-file_info.width_px              #=> 320
-file_info.height_px             #=> 240
-file_info.orientation           #=> :top_left
+matches = FormatParser.parse(File.open("myimage.jpg", "rb"))
+matches.first.nature        #=> :image
+matches.first.format        #=> :jpg
+matches.first.width_px      #=> 320
+matches.first.height_px     #=> 240
+matches.first.orientation   #=> :top_left
 ```
 
-If nothing is detected, the result will be `nil`.
+If you would rather receive only one result, call the gem as follows:
+
+```ruby
+FormatParser.parse(File.open("myimage.jpg", "rb"), returns: :one)
+```
+
+You can also optimize the metadata extraction by providing hints to the gem:
+
+```ruby
+FormatParser.parse(File.open("myimage", "rb"), natures: [:video, :image], formats: [:jpg, :png, :mp4])
+```
+
+## Creating your own parsers
+
+In order to create new parsers, these have to meet two requirements:
+
+1) Instances of the new parser class needs to respond to a `call` method which takes one IO object as an argument and returns some metadata information about its corresponding file or nil otherwise.
+2) Instances of the new parser class needs to respond `natures` and `formats` accessor methods, both returning an array of symbols. A simple DSL is provided to avoid writing those accessors.
+3) The class needs to register itself as a parser.
+
+
+Down below you can find a basic parser implementation:
+
+```ruby
+class BasicParser
+  include FormatParser::DSL # Adds formats and natures methods to the class, which define
+                            # accessor for all the instances.
+  
+  formats :foo, :baz # Indicates which formats it can read.
+  natures :bar       # Indicates which type of file from a human perspective it can read:
+                     #      - :audio
+                     #      - :document
+                     #      - :image
+                     #      - :video
+  def call(file)
+    # Returns a DTO object with including some metadata.
+  end
+
+  FormatParser.register_parser_constructor self # Register this parser.
+```
 
 ## Design rationale
 

data/format_parser.gemspec

@@ -15,8 +15,14 @@ Gem::Specification.new do |spec|
   minimum amount of data possible."
   spec.homepage      = "https://github.com/WeTransfer/format_parser"
   spec.license       = "MIT"
-
-  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # Alert people to a change in the gem's interface, will remove in a subsequent version
+  spec.post_install_message = %q{
+    -----------------------------------------------------------------------------
+    | ATTENTION: format_parser v0.2.0 introduces changes to the gem's interface.|
+    | See https://github.com/WeTransfer/format_parser#basic-usage               |
+    | for up-to-date usage instructions. Thank you for using format_parser! :)  |
+    -----------------------------------------------------------------------------
+  }
   # to allow pushing to a single host or delete this section to allow pushing to any host.
   if spec.respond_to?(:metadata)
     spec.metadata['allowed_push_host'] = "https://rubygems.org"
@@ -35,7 +41,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'ks', '~> 0.0.1'
   spec.add_dependency 'exifr', '~> 1.0'
   spec.add_dependency 'faraday', '~> 0.13'
-  
+
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rake', '~> 12'
   spec.add_development_dependency 'simplecov', '~> 0.15'

data/lib/audio.rb

@@ -0,0 +1,37 @@
+module FormatParser
+  class Audio
+    NATURE = :audio
+
+    # Type of the file (e.g :mp3)
+    attr_accessor :format
+
+    # The number of audio channels for sound files that are muxed
+    # and for video files with embedded sound
+    attr_accessor :num_audio_channels
+
+    # SampeThe number of audio channels for sound files that are muxed
+    # and for video files with embedded sound
+    attr_accessor :audio_sample_rate_hz
+
+    # Duration of the media object (be it audio or video) in seconds,
+    # as a Float
+    attr_accessor :media_duration_seconds
+
+    # Duration of the media object in addressable frames or samples,
+    # as an Integer
+    attr_accessor :media_duration_frames
+
+    # If a parser wants to provide any extra information to the caller
+    # it can be placed here
+    attr_accessor :intrinsics
+
+    # Only permits assignments via defined accessors
+    def initialize(**attributes)
+      attributes.map { |(k, v)| public_send("#{k}=", v) }
+    end
+
+    def nature
+      NATURE
+    end
+  end
+end

data/lib/document.rb

@@ -0,0 +1,17 @@
+module FormatParser
+  class Document
+    NATURE = :document
+
+    attr_accessor :format
+    attr_accessor :document_type
+
+    # Only permits assignments via defined accessors
+    def initialize(**attributes)
+      attributes.map { |(k, v)| public_send("#{k}=", v) }
+    end
+
+    def nature
+      NATURE
+    end
+  end
+end

data/lib/format_parser.rb

@@ -1,12 +1,16 @@
 require 'thread'
 
 module FormatParser
-  require_relative 'file_information'
+  require_relative 'image'
+  require_relative 'audio'
+  require_relative 'document'
+  require_relative 'video'
   require_relative 'io_utils'
   require_relative 'read_limiter'
   require_relative 'remote_io'
   require_relative 'io_constraint'
   require_relative 'care'
+  require_relative 'parsers/dsl'
 
   PARSER_MUX = Mutex.new
 
@@ -14,6 +18,13 @@ module FormatParser
     PARSER_MUX.synchronize do
       @parsers ||= []
       @parsers << object_responding_to_new
+      # Gathering natures and formats from parsers. An instance has to be created.
+      parser = object_responding_to_new.new
+      @natures ||= Set.new
+      # NOTE: merge method for sets modify the instance.
+      @natures.merge(parser.natures)
+      @formats ||= Set.new
+      @formats.merge(parser.formats)
     end
   end
 
@@ -30,25 +41,32 @@ module FormatParser
     parse(cached_io)
   end
 
-  def self.parse(io)
+  def self.parse(io, natures: @natures.to_a, formats: @formats.to_a, returns: :all)
     # If the cache is preconfigured do not apply an extra layer. It is going
     # to be preconfigured when using parse_http.
     io = Care::IOWrapper.new(io) unless io.is_a?(Care::IOWrapper)
 
+    # How many results has the user asked for? Used to determinate whether an array
+    # is returned or not.
+    amount = case returns
+             when :all
+               @parsers.count
+             when :one
+               1
+             else
+               throw ArgumentError.new(":returns does not match any supported mode (:all, :one)")
+             end
+
     # Always instantiate parsers fresh for each input, since they might
     # contain instance variables which otherwise would have to be reset
     # between invocations, and would complicate threading situations
-    parsers = @parsers.map(&:new)
-
-    parsers.each do |parser|
+    results = parsers_for(natures, formats).map do |parser|
       # We need to rewind for each parser, anew
       io.seek(0)
       # Limit how many operations the parser can perform
       limited_io = ReadLimiter.new(io, max_bytes: 512*1024, max_reads: 64*1024, max_seeks: 64*1024)
       begin
-        if info = parser.information_from_io(limited_io)
-          return info
-        end
+        parser.call(limited_io)
       rescue IOUtils::InvalidRead
         # There was not enough data for this parser to work on,
         # and it triggered an error
@@ -57,8 +75,21 @@ module FormatParser
         # caused the parser to go off-track. Strictly speaking we should log this
         # and examine the file more closely.
       end
-    end
-    nil # Nothing matched
+    end.reject(&:nil?).take(amount)
+
+    return results.first if amount == 1
+    # Convert the results from a lazy enumerator to an array.
+    results.to_a
+  end
+
+  private
+
+  def self.parsers_for(natures, formats)
+    # returns lazy enumerator for only computing the minimum amount of work (see :returns keyword argument)
+    @parsers.map(&:new).select do |parser|
+      # Do a given parser contain any nature and/or format asked by the user?
+      (natures & parser.natures).size > 0 && (formats & parser.formats).size > 0
+      end.lazy
   end
 
   Dir.glob(__dir__ + '/parsers/*.rb').sort.each do |parser_file|

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.1.7'
+  VERSION = '0.2.0'
 end

data/lib/{file_information.rb → image.rb}

@@ -1,13 +1,11 @@
 module FormatParser
-  class FileInformation
-
-    # What kind of file is it?
-    attr_accessor :file_nature
+  class Image
+    NATURE = :image
 
     # What filetype was recognized? Will contain a non-ambiguous symbol
     # referring to the file format. The symbol can be used as a filename
     # extension safely
-    attr_accessor :file_type
+    attr_accessor :format
 
     # Number of pixels horizontally in the pixel buffer
     attr_accessor :width_px
@@ -42,25 +40,6 @@ module FormatParser
     # http://magnushoff.com/jpeg-orientation.html
     attr_accessor :image_orientation
 
-    # The number of audio channels for sound files that are muxed
-    # and for video files with embedded sound
-    attr_accessor :num_audio_channels
-
-    # SampeThe number of audio channels for sound files that are muxed
-    # and for video files with embedded sound
-    attr_accessor :audio_sample_rate_hz
-
-    # Duration of the media object (be it audio or video) in seconds,
-    # as a Float
-    attr_accessor :media_duration_seconds
-
-    # Duration of the media object in addressable frames or samples,
-    # as an Integer
-    attr_accessor :media_duration_frames
-
-    # XML Document Type
-    attr_accessor :document_type
-
     # If a parser wants to provide any extra information to the caller
     # it can be placed here
     attr_accessor :intrinsics
@@ -70,8 +49,8 @@ module FormatParser
       attributes.map { |(k, v)| public_send("#{k}=", v) }
     end
 
-    def self.image(**kwargs)
-      new(file_nature: :image, **kwargs)
+    def nature
+      NATURE
     end
   end
 end

data/lib/parsers/aiff_parser.rb

@@ -1,5 +1,6 @@
 class FormatParser::AIFFParser
   include FormatParser::IOUtils
+  include FormatParser::DSL
 
   # Known chunk types we can omit when parsing,
   # grossly lifted from http://www.muratnkonar.com/aiff/
@@ -18,7 +19,10 @@ class FormatParser::AIFFParser
     'ANNO',
   ]
 
-  def information_from_io(io)
+  natures :audio
+  formats :aiff
+
+  def call(io)
     io = FormatParser::IOConstraint.new(io)
     form_chunk_type, chunk_size = safe_read(io, 8).unpack('a4N')
     return unless form_chunk_type == "FORM" && chunk_size > 4
@@ -62,9 +66,8 @@ class FormatParser::AIFFParser
     duration_in_seconds = sample_frames / sample_rate
     return unless duration_in_seconds > 0
 
-    FormatParser::FileInformation.new(
-      file_nature: :audio,
-      file_type: :aiff,
+    FormatParser::Audio.new(
+      format: :aiff,
       num_audio_channels: channels,
       audio_sample_rate_hz: sample_rate.to_i,
       media_duration_frames: sample_frames,

data/lib/parsers/dpx_parser.rb

@@ -1,5 +1,10 @@
 class FormatParser::DPXParser
   include FormatParser::IOUtils
+  include FormatParser::DSL
+
+  natures :image
+  formats :dpx
+
   FILE_INFO = [
 #    :x4,   # magic bytes SDPX, we read them anyway so not in the pattern
     :x4,   # u32  :image_offset,   :desc => 'Offset to image data in bytes', :req => true
@@ -124,7 +129,7 @@ class FormatParser::DPXParser
   LE_MAGIC = BE_MAGIC.reverse
   HEADER_SIZE = SIZEOF[DPX_INFO] # Does not include the initial 4 bytes
 
-  def information_from_io(io)
+  def call(io)
     io = FormatParser::IOConstraint.new(io)
     magic = io.read(4)
 
@@ -133,8 +138,8 @@ class FormatParser::DPXParser
     unpack_pattern = DPX_INFO
     unpack_pattern = DPX_INFO_LE if magic == LE_MAGIC
     num_elements, pixels_per_line, num_lines, *_ = safe_read(io, HEADER_SIZE).unpack(unpack_pattern)
-    FormatParser::FileInformation.image(
-      file_type: :dpx,
+    FormatParser::Image.new(
+      format: :dpx,
       width_px: pixels_per_line,
       height_px: num_lines,
     )

data/lib/parsers/dsl.rb

@@ -0,0 +1,29 @@
+module FormatParser
+  # Small DSL to avoid repetitive code while defining a new parsers. Also, it can be leveraged by
+  # third parties to define their own parsers.
+  module DSL
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def formats(*registred_formats)
+        __define(:formats, registred_formats)
+      end
+
+      def natures(*registred_natures)
+        __define(:natures, registred_natures)
+      end
+
+      private
+
+      def __define(name, value)
+        throw ArgumentError("empty array") if value.empty?
+        throw ArgumentError("requires array of symbols") if value.any? { |s| !s.is_a?(Symbol) }
+        define_method(name) do
+          value
+        end
+      end
+    end
+  end
+end

data/lib/parsers/fdx_parser.rb

@@ -1,26 +1,29 @@
 class FormatParser::FDXParser
   include FormatParser::IOUtils
+  include FormatParser::DSL
 
-  def information_from_io(io)
+  formats :fdx
+  natures :document
+
+  def call(io)
     return if !xml_check(io)
     file_and_document_type = safe_read(io, 100)
     file_type, document_type = check_for_document_type(file_and_document_type)
     return if file_type != :fdx
-    file_info = FormatParser::FileInformation.new(
-      file_nature: :document,
-      file_type: file_type,
+    FormatParser::Document.new(
+      format: file_type,
       document_type: document_type
     )
   end
 
   def xml_check(io)
     xml_check = safe_read(io, 5)
-    xml_check == "<?xml" ? true : false
+    xml_check == "<?xml"
   end
-    
+
   def check_for_document_type(file_and_document_type)
     sanitized_data = file_and_document_type.downcase
-    if sanitized_data.include?("finaldraft") && sanitized_data.include?("script") 
+    if sanitized_data.include?("finaldraft") && sanitized_data.include?("script")
       return :fdx, :script
     else
       return