format_parser 0.1.0

data/lib/file_information.rb

@@ -0,0 +1,70 @@
+module FormatParser
+  class FileInformation
+
+    # What kind of file is it?
+    attr_accessor :file_nature
+
+    # What filetype was recognized? Will contain a non-ambiguous symbol
+    # referring to the file format. The symbol can be used as a filename
+    # extension safely
+    attr_accessor :file_type
+
+    # Number of pixels horizontally in the pixel buffer
+    attr_accessor :width_px
+
+    # Number of pixels vertically in the pixel buffer
+    attr_accessor :height_px
+
+    # Whether the file has multiple frames (relevant for image files and video)
+    attr_accessor :has_multiple_frames
+
+    # The angle by which the camera was rotated when taking the picture
+    # (affects display width and height)
+    attr_accessor :orientation
+
+    # Whether the image has transparency (or an alpha channel)
+    attr_accessor :has_transparency
+
+    # Basic information about the color mode
+    attr_accessor :color_mode
+
+    # If the file has animation or is video, this might
+    # indicate the number of frames. Some formats do not
+    # allow retrieving this value without parsing the entire
+    # file, so for GIF this might be nil even though it is
+    # animated. For a boolean check, `has_multiple_frames`
+    # might offer a better clue.
+    attr_accessor :num_animation_or_video_frames
+
+    # Orientation from EXIF data. Will come through as an integer.
+    # To be perfectly honest EXIF orientation handling is a bit of a mess,
+    # here's a reasonable blog post about it:
+    # http://magnushoff.com/jpeg-orientation.html
+    attr_accessor :image_orientation
+
+    # The number of audio channels for sound files that are muxed
+    # and for video files with embedded sound
+    attr_accessor :num_audio_channels
+
+    # SampeThe number of audio channels for sound files that are muxed
+    # and for video files with embedded sound
+    attr_accessor :audio_sample_rate_hz
+
+    # Duration of the media object (be it audio or video) in seconds,
+    # as a Float
+    attr_accessor :media_duration_seconds
+
+    # Duration of the media object in addressable frames or samples,
+    # as an Integer
+    attr_accessor :media_duration_frames
+
+    # Only permits assignments via defined accessors
+    def initialize(**attributes)
+      attributes.map { |(k, v)| public_send("#{k}=", v) }
+    end
+
+    def self.image(**kwargs)
+      new(file_nature: :image, **kwargs)
+    end
+  end
+end

data/lib/format_parser.rb

@@ -0,0 +1,55 @@
+require 'thread'
+
+module FormatParser
+  require_relative 'file_information'
+  require_relative 'io_utils'
+  require_relative 'read_limiter'
+  require_relative 'remote_io'
+  require_relative 'care'
+
+  PARSER_MUX = Mutex.new
+
+  def self.register_parser_constructor(object_responding_to_new)
+    PARSER_MUX.synchronize do
+      @parsers ||= []
+      @parsers << object_responding_to_new
+    end
+  end
+
+  def self.parse_http(url)
+    parse(RemoteIO.new(url))
+  end
+
+  def self.parse(io)
+    io = Care::IOWrapper.new(io) unless io.is_a?(Care::IOWrapper)
+
+    # Always instantiate parsers fresh for each input, since they might
+    # contain instance variables which otherwise would have to be reset
+    # between invocations, and would complicate threading situations
+    parsers = @parsers.map(&:new)
+
+    parsers.each do |parser|
+      # We need to rewind for each parser, anew
+      io.seek(0)
+      # Limit how many operations the parser can perform
+      limited_io = ReadLimiter.new(io, max_bytes: 512*1024, max_reads: 64*1024, max_seeks: 64*1024)
+      begin
+        if info = parser.information_from_io(limited_io)
+          return info
+        end
+      rescue IOUtils::InvalidRead
+        # There was not enough data for this parser to work on,
+        # and it triggered an error
+      rescue ReadLimiter::BudgetExceeded
+        # The parser tried to read too much - most likely the file structure
+        # caused the parser to go off-track. Strictly speaking we should log this
+        # and examine the file more closely.
+      end
+    end
+    nil # Nothing matched
+  end
+
+  Dir.glob(__dir__ + '/parsers/*.rb').sort.each do |parser_file|
+    require parser_file
+  end
+end

data/lib/format_parser/version.rb

@@ -0,0 +1,3 @@
+module FormatParser
+  VERSION = '0.1.0'
+end

data/lib/io_utils.rb

@@ -0,0 +1,41 @@
+module FormatParser::IOUtils
+  class InvalidRead < ArgumentError
+  end
+
+  def safe_read(io, n)
+    if n.nil?
+      raise ArgumentError, "Unbounded reads are not supported"
+    end
+    buf = io.read(n)
+
+    if !buf
+      raise InvalidRead, "We wanted to read #{n} bytes from the IO, but the IO is at EOF"
+    end
+    if buf.bytesize != n
+      raise InvalidRead, "We wanted to read #{n} bytes from the IO, but we got #{buf.bytesize} instead"
+    end
+
+    buf
+  end
+
+  def safe_skip(io, n)
+    if n.nil?
+      raise ArgumentError, "Unbounded skips are not supported"
+    end
+
+    return if n == 0
+
+    if n < 0
+      raise InvalidRead, "Negative skips are not supported"
+    end
+
+    if io.respond_to?(:pos)
+      io.seek(io.pos + n)
+    else
+      safe_read(io, n)
+    end
+    nil
+  end
+
+  ### TODO: Some kind of built-in offset for the read
+end

data/lib/parsers/aiff_parser.rb

@@ -0,0 +1,86 @@
+class FormatParser::AIFFParser
+  include FormatParser::IOUtils
+
+  # Known chunk types we can omit when parsing,
+  # grossly lifted from http://www.muratnkonar.com/aiff/
+  KNOWN_CHUNKS = [
+    'COMT',
+    'INST',
+    'MARK',
+    'SKIP',
+    'SSND',
+    'MIDI',
+    'AESD',
+    'APPL',
+    'NAME',
+    'AUTH',
+    '(c) ', # yes it is a thing
+    'ANNO',
+  ]
+
+  def information_from_io(io)
+    io.seek(0)
+    form_chunk_type, chunk_size = safe_read(io, 8).unpack('a4N')
+    return unless form_chunk_type == "FORM" && chunk_size > 4
+
+    fmt_chunk_type = safe_read(io, 4)
+    
+    return unless fmt_chunk_type == "AIFF"
+
+    # There might be COMT chunks, for example in Logic exports
+    loop do
+      chunk_type, chunk_size = safe_read(io, 8).unpack('a4N')
+      case chunk_type
+      when 'COMM'
+        # The ID is always COMM. The chunkSize field is the number of bytes in the
+        # chunk. This does not include the 8 bytes used by ID and Size fields. For
+        # the Common Chunk, chunkSize should always 18 since there are no fields of
+        # variable length (but to maintain compatibility with possible future
+        # extensions, if the chunkSize is > 18, you should always treat those extra
+        # bytes as pad bytes).
+        return unpack_comm_chunk(io)
+      when *KNOWN_CHUNKS
+        # We continue looping only if we encountered something that looks like
+        # a valid AIFF chunk type - skip the size and continue
+        safe_skip(io, chunk_size)
+        next
+      else # This most likely not an AIFF
+        return nil
+      end
+    end
+  end
+
+  def unpack_comm_chunk(io)
+    # Parse the COMM chunk
+    channels, sample_frames, sample_size, sample_rate_extended = safe_read(io, 2 + 4 + 2 + 10).unpack('nNna10')
+    sample_rate = unpack_extended_float(sample_rate_extended)
+    bytes_per_sample = (sample_size - 1) / 8 + 1
+
+    return unless sample_frames > 0
+
+    # The sample rate is in Hz, so to get duration in seconds, as a float...
+    duration_in_seconds = sample_frames / sample_rate
+    return unless duration_in_seconds > 0
+
+    FormatParser::FileInformation.new(
+      file_nature: :audio,
+      file_type: :aiff,
+      num_audio_channels: channels,
+      audio_sample_rate_hz: sample_rate.to_i,
+      media_duration_frames: sample_frames,
+      media_duration_seconds: duration_in_seconds,
+    )
+  end
+ 
+  def unpack_extended_float(ten_bytes_string)
+    extended = ten_bytes_string.unpack('B80')[0]
+
+    sign = extended[0, 1]
+    exponent = extended[1, 15].to_i(2) - ((1 << 14) - 1)
+    fraction = extended[16, 64].to_i(2)
+  
+    ((sign == '1') ? -1.0 : 1.0) * (fraction.to_f / ((1 << 63) - 1)) * (2 ** exponent)
+  end
+
+  FormatParser.register_parser_constructor self
+end

data/lib/parsers/dpx_parser.rb

@@ -0,0 +1,143 @@
+class FormatParser::DPXParser
+  include FormatParser::IOUtils
+  FILE_INFO = [
+#    :x4,   # magic bytes SDPX, we read them anyway so not in the pattern
+    :x4,   # u32  :image_offset,   :desc => 'Offset to image data in bytes', :req => true
+    :x8,   # char :version, 8,     :desc => 'Version of header format', :req => true
+    :x4,   # u32  :file_size,      :desc => "Total image size in bytes", :req => true
+    :x4,   # u32  :ditto_key,      :desc => 'Whether the basic headers stay the same through the sequence (1 means they do)'
+    :x4,   # u32  :generic_size,   :desc => 'Generic header length'
+    :x4,   # u32  :industry_size,  :desc => 'Industry header length'
+    :x4,   # u32  :user_size,      :desc => 'User header length'
+    :x100, # char :filename, 100,  :desc => 'Original filename'
+    :x24,  # char :timestamp, 24,  :desc => 'Creation timestamp'
+    :x100, # char :creator, 100,   :desc => 'Creator application'
+    :x200, # char :project, 200,   :desc => 'Project name'
+    :x200, # char :copyright, 200, :desc => 'Copyright'
+    :x4,   # u32  :encrypt_key,   :desc => 'Encryption key'
+    :x104,  # blanking :reserve, 104
+  ].join
+
+  FILM_INFO = [
+    :x2, # char :id, 2,          :desc => 'Film mfg. ID code (2 digits from film edge code)'
+    :x2, # char :type, 2,        :desc => 'Film type (2 digits from film edge code)'
+    :x2, # char :offset, 2,      :desc => 'Offset in perfs (2 digits from film edge code)'
+    :x6, # char :prefix, 6,      :desc => 'Prefix (6 digits from film edge code'
+    :x4, # char :count, 4,       :desc => 'Count (4 digits from film edge code)'
+    :x32,# char :format, 32,     :desc => 'Format (e.g. Academy)'
+    :x4, # u32 :frame_position,  :desc => 'Frame position in sequence'
+    :x4, # u32 :sequence_extent, :desc => 'Sequence length'
+    :x4, # u32 :held_count,      :desc => 'For how many frames the frame is held'
+    :x4, # r32 :frame_rate,      :desc => 'Frame rate'
+    :x4, # r32 :shutter_angle,   :desc => 'Shutter angle'
+    :x4, # char :frame_id, 32,   :desc => 'Frame identification (keyframe)'
+    :x4, # char :slate, 100,     :desc => 'Slate information'
+    :x4, # blanking :reserve, 56
+  ].join
+
+  IMAGE_ELEMENT = [
+    :x4, # u32 :data_sign, :desc => 'Data sign (0=unsigned, 1=signed). Core is unsigned', :req => true
+    #
+    :x4, # u32 :low_data,      :desc => 'Reference low data code value'
+    :x4, # r32 :low_quantity,  :desc => 'Reference low quantity represented'
+    :x4, # u32 :high_data,     :desc => 'Reference high data code value (1023 for 10bit per channel)'
+    :x4, # r32 :high_quantity, :desc => 'Reference high quantity represented'
+    #
+    :x1, # u8 :descriptor,   :desc => 'Descriptor for this image element (ie Video or Film), by enum', :req => true
+    # TODO - colirimetry information might be handy to recover,
+    # as well as "bit size per element" (how many bits _per component_ we have) -
+    # this will be different for, say, 8-bit DPX files versus 10-bit etc.
+    :x1, # u8 :transfer,     :desc => 'Transfer function (ie Linear), by enum', :req => true
+    :x1, # u8 :colorimetric, :desc => 'Colorimetric (ie YcbCr), by enum', :req => true
+    :x1, # u8 :bit_size,     :desc => 'Bit size for element (ie 10)', :req => true
+    #
+    :x2, # u16 :packing,     :desc => 'Packing (0=Packed into 32-bit words, 1=Filled to 32-bit words))', :req => true
+    :x2, # u16 :encoding,    :desc => "Encoding (0=None, 1=RLE)", :req => true
+    :x4, # u32 :data_offset, :desc => 'Offset to data for this image element', :req => true
+    :x4, # u32 :end_of_line_padding, :desc => "End-of-line padding for this image element"
+    :x4, # u32 :end_of_image_padding, :desc => "End-of-line padding for this image element"
+    :x32,# char :description, 32
+  ].join
+
+  IMAGE_INFO = [
+    :x2, # u16 :orientation, OrientationInfo,    :desc => 'Orientation descriptor',    :req => true
+    :n1, # u16 :number_elements,                   :desc => 'How many elements to scan', :req => true
+    :N1, # u32 :pixels_per_line,                   :desc => 'Pixels per horizontal line', :req => true
+    :N1, # u32 :lines_per_element,                 :desc => 'Line count', :req => true
+    IMAGE_ELEMENT * 8, # 8 IMAGE_ELEMENT structures
+    :x52, # blanking :reserve, 52
+  ].join
+
+  ORIENTATION_INFO = [
+    :x4, #  u32 :x_offset
+    :x4, #  u32 :y_offset
+    #
+    :x4, #  r32 :x_center
+    :x4, #  r32 :y_center
+    #
+    :x4, #  u32 :x_size, :desc => 'Original X size'
+    :x4, #  u32 :y_size, :desc => 'Original Y size'
+    #
+    :x100, #  char :filename, 100, :desc => "Source image filename"
+    :x24,  #  char :timestamp, 24, :desc => "Source image/tape timestamp"
+    :x32,  #  char :device,    32, :desc => "Input device or tape"
+    :x32,  #  char :serial,    32, :desc => "Input device serial number"
+    #
+    :x4,   #  array :border, :u16, 4, :desc => 'Border validity: XL, XR, YT, YB'
+    :x4,
+    :x4,
+    :x4,
+
+    # TODO - the aspect ratio might be handy to recover since it
+    # will be used in DPX files in, say, anamorphic (non-square pixels)
+    :x4,   #  array :aspect_ratio , :u32, 2, :desc => "Aspect (H:V)"
+    :x4,
+    #
+    :x28,  #  blanking :reserve, 28
+  ].join
+
+  DPX_INFO = [
+    FILE_INFO,
+    IMAGE_INFO,
+    ORIENTATION_INFO,
+  ].join
+
+  DPX_INFO_LE = DPX_INFO.tr("n", "v").tr("N", "V")
+
+  SIZEOF = ->(pattern) {
+    bytes_per_element = {
+      "v" => 2, # 16bit uints
+      "n" => 2,
+      "V" => 4, # 32bit uints
+      "N" => 4,
+      "C" => 1,
+      "x" => 1,
+    }
+    pattern.scan(/[^\d]\d+/).map do |pattern|
+      unpack_code = pattern[0]
+      num_repetitions = pattern[1..-1].to_i
+      bytes_per_element.fetch(unpack_code) * num_repetitions
+    end.inject(&:+)
+  }
+
+  BE_MAGIC = 'SDPX'
+  LE_MAGIC = BE_MAGIC.reverse
+  HEADER_SIZE = SIZEOF[DPX_INFO] # Does not include the initial 4 bytes
+
+  def information_from_io(io)
+    magic = io.read(4)
+
+    return nil unless [BE_MAGIC, LE_MAGIC].include?(magic)
+
+    unpack_pattern = DPX_INFO
+    unpack_pattern = DPX_INFO_LE if magic == LE_MAGIC
+    num_elements, pixels_per_line, num_lines, *_ = safe_read(io, HEADER_SIZE).unpack(unpack_pattern)
+    FormatParser::FileInformation.image(
+      file_type: :dpx,
+      width_px: pixels_per_line,
+      height_px: num_lines,
+    )
+  end
+
+  FormatParser.register_parser_constructor self
+end

data/lib/parsers/exif_parser.rb

@@ -0,0 +1,58 @@
+require 'exifr/jpeg'
+require 'exifr/tiff'
+
+class FormatParser::EXIFParser
+  include FormatParser::IOUtils
+
+  # Squash exifr's invalid date warning since we do not use that data.
+  logger = Logger.new(nil)
+  EXIFR.logger = logger
+
+  attr_accessor :exif_data, :orientation, :width, :height
+
+  ORIENTATIONS = [
+    :top_left,
+    :top_right,
+    :bottom_right,
+    :bottom_left,
+    :left_top,
+    :right_top,
+    :right_bottom,
+    :left_bottom
+  ]
+
+  def initialize(filetype, file_data)
+    @filetype = filetype
+    @file_data = file_data
+    @exif_data = nil
+    @orientation = nil
+    @height = nil
+    @width = nil
+  end
+
+  def scan_image_exif
+
+    # Without the magic bytes EXIFR throws an error
+    @file_data.rewind
+    raw_exif_data = EXIFR::JPEG.new(@file_data) if @filetype == :jpeg
+    raw_exif_data = EXIFR::TIFF.new(@file_data) if @filetype == :tiff
+    # For things that we don't yet have a parser for
+    # we make the raw exif result available
+    @exif_data = raw_exif_data
+    @orientation = orientation_parser(raw_exif_data)
+    @width = @exif_data.width
+    @height = @exif_data.height
+  end
+
+  def orientation_parser(raw_exif_data)
+    value = raw_exif_data.orientation.to_i
+    if valid_orientation?(value)
+      @orientation = ORIENTATIONS[value - 1]
+    end
+  end
+
+  def valid_orientation?(value)
+    (1..ORIENTATIONS.length).include?(value)
+  end
+
+end