format_parser 2.6.0 → 2.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ce8a7c3fd258ccf2abcee6ea8731c0337e7255dfb189ccfb5fdfc02e9dd9b36
-  data.tar.gz: a6ff6ea6e771f2636e30cfee70b3c22584cb85e7020a97483f66e2c482048444
+  metadata.gz: 5d2679a365f7c735d2b8c962765c4783b6336bf23461ffde1705ab141d250591
+  data.tar.gz: e1e4b4caa2956cbf1653d39498a9db84589cd0c9c979c6d84e9f3b3027427274
 SHA512:
-  metadata.gz: e27bd51913d3a3b3d061ec27379acc4a15fa19ee2e16cccdbf5a0aec4daf8fe26cdfc0062115119616384ded92c645e4a88a6d0244b47b33c5fc79903ccd3906
-  data.tar.gz: 024756cdb460347f36cd1aa247b04f5468982b4c502cf049e8c8abd5cb1020f960006f552c7659770c6149ff6ed3a61def2beb44160236f35fe35533579f6c31
+  metadata.gz: 52e775ae2a4ced22d2879fff48108974bfe4087333b70d74c5c0910229aa7298826664bbd474dd9b4221777637cc51bf969735a830df976f58f0ff7302dd69c5
+  data.tar.gz: 31b8f95ff8fbcba01d60fe2377699a9abebc61a28d74afc05aa8456d6660f786ee268c2a621c0dc83490e3f9b04dc5e1a60319ae6f4c54503b3dfab9d39d520e

data/README.md

@@ -26,6 +26,7 @@ and [dimensions,](https://github.com/sstephenson/dimensions) borrowing from them
 * HEIC
 * HEIF
 * JPEG
+* JSON
 * M3U
 * M4A
 * M4B
@@ -216,7 +217,7 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
 - NEF examples are downloaded from http://www.rawsamples.ch/ and are Creative Common Licensed.
 
 ### OGG
-- `hi.ogg`,  `vorbis.ogg`, `with_confusing_magic_string.ogg`, `with_garbage_at_the_end.ogg` have been generated by the project contributors
+- `hi.ogg`,  `vorbis.ogg`, `with_confusing_magic_string.ogg`, `invalid_with_garbage_at_the_end.ogg` have been generated by the project contributors
 
 ### PDF
 - PDF 2.0 files downloaded from the [PDF Association public Github repository](https://github.com/pdf-association/pdf20examples). These files are licensed under the Creative Commons Attribution-ShareAlike 4.0 International (CC BY-SA 4.0) license.
@@ -235,7 +236,7 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
 ### WAV
 - c_11k16bitpcm.wav and c_8kmp316.wav are from [Wikipedia WAV](https://en.wikipedia.org/wiki/WAV#Comparison_of_coding_schemes), retrieved January 7, 2018
 - c_39064__alienbomb__atmo-truck.wav is from [freesound](https://freesound.org/people/alienbomb/sounds/39064/) and is CC0 licensed
-- c_M1F1-Alaw-AFsp.wav and d_6_Channel_ID.wav are from a [McGill Engineering site](http://www-mmsp.ece.mcgill.ca/Documents/AudioFormats/WAVE/Samples.html)
+- c_M1F1-Alaw-AFsp.wav and invalid_d_6_Channel_ID.wav are from a [McGill Engineering site](http://www-mmsp.ece.mcgill.ca/Documents/AudioFormats/WAVE/Samples.html)
 
 ### WEBP
 - With the exception of extended-animation.webp, which was obtained from Wikimedia Commons and is Creative Commons

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '2.6.0'
+  VERSION = '2.7.1'
 end

data/lib/format_parser.rb

@@ -17,6 +17,7 @@ module FormatParser
   require_relative 'read_limits_config'
   require_relative 'remote_io'
   require_relative 'io_constraint'
+  require_relative 'utf8_reader'
   require_relative 'care'
   require_relative 'active_storage/blob_analyzer'
   require_relative 'text'
@@ -35,6 +36,9 @@ module FormatParser
   # The value will ensure the parser having it will be applied to the file last.
   LEAST_PRIORITY = 99
 
+  @registered_natures = []
+  @registered_formats = []
+
   # Register a parser object to be used to perform file format detection. Each parser FormatParser
   # provides out of the box registers itself using this method.
   #
@@ -67,9 +71,20 @@ module FormatParser
       end
       @parser_priorities ||= {}
       @parser_priorities[callable_parser] = priority
+
+      @registered_natures |= parser_provided_natures
+      @registered_formats |= parser_provided_formats
     end
   end
 
+  def self.registered_natures
+    @registered_natures
+  end
+
+  def self.registered_formats
+    @registered_formats
+  end
+
   # Deregister a parser object (makes FormatParser forget this parser existed). Is mostly used in
   # tests, but can also be used to forcibly disable some formats completely.
   #

data/lib/parsers/json_parser/validator.rb

@@ -0,0 +1,319 @@
+##
+# This class checks whether a given file is a valid JSON file.
+# The validation process DOES NOT assemble an object with the contents of the JSON file in memory,
+# Instead, it implements a simple state-machine-like that digests the contents of the file while traversing
+# the hierarchy of nodes in the document.
+#
+# Although this is based on the IETF standard (https://www.rfc-editor.org/rfc/rfc8259),
+# it does cut a few corners for the sake of simplicity. For instance, instead of validating
+# Numbers, "true", "false" and "null" tokens, it supports a type called Literal to hold generic sequences of characters.
+# This decision makes the implementation simpler while being a good-enough approach to identify JSON files.
+#
+# There is also a cap. Large files are not read all the way through. Instead, if the beginning of file is
+# JSON-compliant, it is assumed that the file is a JSON file.
+
+class FormatParser::JSONParser::Validator
+  class JSONParserError < StandardError
+  end
+
+  MAX_SAMPLE_SIZE = 1024
+  MAX_LITERAL_SIZE = 30 # much larger then necessary.
+  ESCAPE_CHAR = "\\"
+  WHITESPACE_CHARS = [" ", "\t", "\n", "\r"]
+  ENDING_VALUE_CHARS = [",", "]", "}"]
+  LITERALS_CHAR_TEMPLATE = /\w|[+\-.]/ # any alphanumeric, "+", "-" and "."
+
+  def initialize(io)
+    @io = io
+    @current_node = nil # :object, :array, :string, :literal
+    @parent_nodes = []
+    @current_state = :awaiting_root_node
+    @escape_next = false
+    @current_literal_size = 0
+    @pos = 0
+
+    @all_parsers = {}
+
+    @execution_stats = {
+      array: 0,
+      object: 0,
+      literal: 0,
+      string: 0
+    }
+
+    setup_transitions
+  end
+
+  def validate
+    char_reader = FormatParser::UTF8Reader.new(@io)
+
+    while (c = char_reader.read_char)
+      @pos += 1
+      parse_char c
+
+      # Halt validation if the sampling limit is reached.
+      if @pos >= MAX_SAMPLE_SIZE
+        raise JSONParserError, "Invalid JSON file" if @current_state == :awaiting_root_node
+        return false
+      end
+    end
+
+    # Raising error in case the EOF is reached earlier than expected
+    raise JSONParserError, "Incomplete JSON file" if @current_state != :closed
+    true
+  rescue  FormatParser::UTF8Reader::UTF8CharReaderError
+    raise JSONParserError, "Invalid UTF-8 character"
+  end
+
+  def stats(node_type)
+    @execution_stats[node_type]
+  end
+
+  private
+
+  def setup_transitions
+    when_its :awaiting_root_node, ->(c) do
+      read_whitespace(c) or
+        start_object(c) or
+        start_array(c)
+    end
+
+    when_its :awaiting_object_attribute_key, ->(c) do
+      read_whitespace(c) or
+        start_attribute_key(c) or
+        close_object(c)
+    end
+
+    when_its :reading_object_attribute_key, ->(c) do
+      close_attribute_key(c) or
+        read_valid_string_char(c)
+    end
+
+    when_its :awaiting_object_colon_separator, ->(c) do
+      read_whitespace(c) or
+        read_colon(c)
+    end
+
+    when_its :awaiting_object_attribute_value, ->(c) do
+      read_whitespace(c) or
+        start_object(c) or
+        start_array(c) or
+        start_string(c) or
+        start_literal(c)
+    end
+
+    when_its :awaiting_array_value, ->(c) do
+      read_whitespace(c) or
+        start_object(c) or
+        start_array(c) or
+        start_string(c) or
+        start_literal(c) or
+        close_array(c)
+    end
+
+    when_its :reading_string, ->(c) do
+      close_string(c) or
+        read_valid_string_char(c)
+    end
+
+    when_its :awaiting_next_or_close, ->(c) do
+      read_whitespace(c) or
+        read_comma_separator(c) or
+        close_object(c) or
+        close_array(c)
+    end
+
+    when_its :reading_literal, ->(c) do
+      read_valid_literal_char(c) or (
+        close_literal(c) and (
+          read_whitespace(c) or
+          read_comma_separator(c) or
+          close_array(c) or
+          close_object(c)))
+    end
+
+    when_its :closed, ->(c) do
+      read_whitespace(c)
+    end
+  end
+
+  def when_its(state, act)
+    @all_parsers[state] = act
+  end
+
+  def parse_char(c)
+    next_step = @all_parsers[@current_state]
+    accepted = next_step.call(c)
+    reject_char(c) unless accepted
+  end
+
+  def read_whitespace(c)
+    whitespace?(c)
+  end
+
+  def read_colon(c)
+    if c == ":"
+      @current_state = :awaiting_object_attribute_value
+      return true
+    end
+    false
+  end
+
+  def read_valid_string_char(c)
+    if @escape_next
+      @escape_next = false
+      return true
+    end
+
+    if c == ESCAPE_CHAR
+      @escape_next = true
+      return true
+    end
+    !control_char?(c) and c != "\""
+  end
+
+  def read_valid_literal_char(c)
+    if valid_literal_char?(c)
+      @current_literal_size += 1
+      return true
+    end
+
+    false
+  end
+
+  def read_comma_separator(c)
+    if c == ","
+      @current_state = :awaiting_object_attribute_key if @current_node == :object
+      @current_state = :awaiting_array_value if @current_node == :array
+      return true
+    end
+    false
+  end
+
+  # Object: {"k1":"val", "k2":[1,2,3], "k4": undefined, "k5": {"l1": 6}}
+  def start_object(c)
+    return false if whitespace?(c)
+    return false unless c == "{"
+
+    begin_node(:object)
+    @current_state = :awaiting_object_attribute_key
+    true
+  end
+
+  def close_object(c)
+    return false if whitespace?(c)
+    return false unless @current_node == :object and c == "}"
+
+    end_node
+    @current_state = :awaiting_next_or_close unless @current_node.nil?
+    true
+  end
+
+  # Array: [1, "two", true, undefined, {}, []]
+  def start_array(c)
+    return false unless c == "["
+
+    begin_node(:array)
+    @current_state = :awaiting_array_value
+    true
+  end
+
+  def close_array(c)
+    return false if whitespace?(c)
+    return false unless @current_node == :array and c == "]"
+
+    end_node
+    @current_state = :awaiting_next_or_close unless @current_node.nil?
+    true
+  end
+
+  def start_attribute_key(c)
+    return false unless c == "\""
+
+    begin_node(:string)
+    @current_state = :reading_object_attribute_key
+    true
+  end
+
+  def close_attribute_key(c)
+    return false if @escape_next
+    return false unless c == "\""
+    end_node
+    @current_state = :awaiting_object_colon_separator
+    true
+  end
+
+  # Strings: "Foo"
+  def start_string(c)
+    return false unless c == "\""
+
+    begin_node(:string)
+    @current_state = :reading_string
+    true
+  end
+
+  def close_string(c)
+    return false if @escape_next
+    return false unless c == "\""
+    end_node
+    @current_state = :awaiting_next_or_close
+    true
+  end
+
+  # literals: null, undefined, true, false, NaN, infinity, -123.456e10 -123,456e10
+  def start_literal(c)
+    return false unless valid_literal_char?(c)
+
+    begin_node(:literal)
+    @current_state = :reading_literal
+    @current_literal_size = 1
+    true
+  end
+
+  def close_literal(c)
+    raise JSONParserError, "Literal to large at #{@pos}" if @current_literal_size > MAX_LITERAL_SIZE
+
+    if whitespace?(c) || ENDING_VALUE_CHARS.include?(c)
+      end_node
+      @current_state = :awaiting_next_or_close
+      return true
+    end
+
+    false
+  end
+
+  # Marks the creation of a node (object, array, string or literal)
+  def begin_node(node_type)
+    # Accounting for the new node
+    @execution_stats[node_type] ||= 0
+    @execution_stats[node_type] += 1
+
+    # Managing the node execution stack
+    @parent_nodes.push(@current_node)
+    @current_node = node_type
+  end
+
+  # Marks the closure of a node (object, array, string or literal)
+  def end_node
+    @current_node = @parent_nodes.pop
+    @current_state = :closed if @current_node.nil?
+  end
+
+  def reject_char(char)
+    raise JSONParserError, "Unexpected char #{char} in position #{@pos}"
+  end
+
+  def whitespace?(c)
+    WHITESPACE_CHARS.include?(c)
+  end
+
+  def control_char?(c)
+    # control characters: (U+0000 through U+001F)
+    utf8_code = c.unpack('U*')[0]
+    utf8_code <= 31
+  end
+
+  def valid_literal_char?(c)
+    LITERALS_CHAR_TEMPLATE === c
+  end
+end

data/lib/parsers/json_parser.rb

@@ -0,0 +1,25 @@
+class FormatParser::JSONParser
+  include FormatParser::IOUtils
+  require_relative 'json_parser/validator'
+
+  JSON_MIME_TYPE = 'application/json'
+
+  def likely_match?(filename)
+    filename =~ /\.json$/i
+  end
+
+  def call(io)
+    io = FormatParser::IOConstraint.new(io)
+    validator = Validator.new(io)
+
+    validator.validate
+
+    FormatParser::Text.new(
+      format: :json,
+      content_type: JSON_MIME_TYPE,
+    )
+  rescue Validator::JSONParserError
+    nil
+  end
+  FormatParser.register_parser new, natures: :text, formats: :json
+end

data/lib/parsers/mp3_parser.rb

@@ -76,6 +76,11 @@ class FormatParser::MP3Parser
     io.seek(0)
     return if TIFF_HEADER_BYTES.include?(safe_read(io, 4))
 
+    # Prevention against parsing WAV files.
+    io.seek(0)
+    wav_chunk_id, _wav_size, wav_riff_type = safe_read(io, 12).unpack('a4la4')
+    return if wav_chunk_id == 'RIFF' || wav_riff_type == 'WAVE'
+
     # Read all the ID3 tags (or at least attempt to)
     io.seek(0)
     id3v1 = ID3Extraction.attempt_id3_v1_extraction(io)
@@ -315,5 +320,5 @@ class FormatParser::MP3Parser
     end
   end
 
-  FormatParser.register_parser new, natures: :audio, formats: :mp3, priority: 99
+  FormatParser.register_parser new, natures: :audio, formats: :mp3, priority: 101
 end

data/lib/utf8_reader.rb

@@ -0,0 +1,68 @@
+##
+# This class Reads individual characters from files using UTF-8 encoding
+# This deals with two main concerns:
+#   - Variable byte length of characters
+#   - Reducing the number of read operations by loading bytes in chunks
+
+class FormatParser::UTF8Reader
+  READ_CHUNK_SIZE = 128
+
+  class UTF8CharReaderError < StandardError
+  end
+
+  def initialize(io)
+    @io = io
+    @chunk = ""
+    @index = 0
+    @eof = false
+  end
+
+  def read_char
+    first_byte = read_byte
+    return if first_byte.nil?
+
+    char_length = assess_char_length(first_byte)
+    as_bytes = Array.new(char_length) do |i|
+      next first_byte if i == 0
+      read_byte
+    end
+
+    char = as_bytes.pack('c*').force_encoding('UTF-8')
+    raise UTF8CharReaderError, "Invalid UTF-8 character" unless char.valid_encoding?
+
+    char
+  rescue TypeError
+    raise UTF8CharReaderError, "Invalid UTF-8 character"
+  end
+
+  private
+
+  def read_byte
+    manage_data_chunk
+    return if @chunk.nil?
+    byte = @chunk.bytes[@index]
+    @index += 1 unless byte.nil?
+    byte
+  end
+
+  def manage_data_chunk
+    return if @index < @chunk.length
+    @chunk = @io.read(READ_CHUNK_SIZE)
+    @chunk ||= ""
+    @index = 0
+    @eof = true if @chunk.nil? or @chunk.length < READ_CHUNK_SIZE
+  end
+
+  def assess_char_length(first_byte)
+    # 0_______ (1 byte)
+    # 110_____ (2 bytes) 192
+    # 1110____ (3 bytes) 224
+    # 11110___ (4 bytes) 240
+    case first_byte
+    when 240.. then 4
+    when 224..239 then 3
+    when 192..223 then 2
+    else 1
+    end
+  end
+end

data/spec/format_parser_spec.rb

@@ -34,6 +34,26 @@ describe FormatParser do
       end
     end
 
+    it "fixtures with 'invalid' in the filename should fail to parse" do
+      Dir.glob(fixtures_dir + '/**/*.*').sort.each do |fixture_path|
+        file_name = File.basename(fixture_path)
+        next unless file_name.include? "invalid"
+        File.open(fixture_path, 'rb') do |file|
+          FormatParser.parse(file)
+        end
+      end
+    end
+
+    it "fixtures without 'invalid' in the filename should be parsed successfully" do
+      Dir.glob(fixtures_dir + '/**/*.*').sort.each do |fixture_path|
+        file_name = File.basename(fixture_path)
+        next if file_name.include? "invalid"
+        File.open(fixture_path, 'rb') do |file|
+          FormatParser.parse(file)
+        end
+      end
+    end
+
     it 'triggers parsers in a certain order that corresponds to the parser priorities' do
       file_contents = StringIO.new('a' * 4096)
 
@@ -189,12 +209,20 @@ describe FormatParser do
         'FormatParser::CR3Parser',
         'FormatParser::DPXParser',
         'FormatParser::FLACParser',
-        'FormatParser::MP3Parser',
         'FormatParser::OggParser',
         'FormatParser::TIFFParser',
-        'FormatParser::WAVParser'
+        'FormatParser::WAVParser',
+        'FormatParser::MP3Parser'
       ])
     end
+
+    it 'ensures that MP3 parser is the last one among all' do
+      natures = FormatParser.registered_natures
+      formats = FormatParser.registered_formats
+      prioritised_parsers = FormatParser.parsers_for(natures, formats)
+      parser_class_names = prioritised_parsers.map { |parser| parser.class.name }
+      expect(parser_class_names.last).to eq 'FormatParser::MP3Parser'
+    end
   end
 
   describe '.register_parser and .deregister_parser' do

data/spec/parsers/flac_parser_spec.rb

@@ -55,7 +55,7 @@ describe FormatParser::FLACParser do
   end
 
   it 'raises an error when sample rate is 0' do
-    fpath = fixtures_dir + 'FLAC/sample_rate_0.flac'
+    fpath = fixtures_dir + 'FLAC/invalid_sample_rate_0.flac'
 
     expect {
       subject.call(File.open(fpath, 'rb'))

data/spec/parsers/json_parser/validator_spec.rb

@@ -0,0 +1,321 @@
+require 'spec_helper'
+
+describe FormatParser::JSONParser::Validator do
+  def load_file(file_name)
+    io = File.open(Pathname.new(fixtures_dir).join('JSON').join(file_name), 'rb')
+    FormatParser::JSONParser::Validator.new(io)
+  end
+
+  def load_string(content)
+    io = StringIO.new(content.encode(Encoding::UTF_8))
+    FormatParser::JSONParser::Validator.new(io)
+  end
+
+  describe 'When reading root nodes' do
+    it "identifies objects as root nodes" do
+      v = load_string '{"key": "value"}'
+
+      completed = v.validate
+
+      expect(completed).to be true
+      expect(v.stats(:object)).to be 1
+      expect(v.stats(:string)).to be 2
+    end
+
+    it "identifies arrays as root nodes" do
+      v = load_string '["e1", "e2"]'
+
+      completed = v.validate
+
+      expect(completed).to be true
+      expect(v.stats(:array)).to be 1
+      expect(v.stats(:string)).to be 2
+    end
+
+    it "rejects strings as root nodes" do
+      expect do
+        v = load_string '"this is a string"'
+        v.validate
+        end.to raise_error(FormatParser::JSONParser::Validator::JSONParserError)
+    end
+
+    it "rejects literals as root nodes" do
+      expect do
+        v = load_string 'true'
+        v.validate
+        end.to raise_error(FormatParser::JSONParser::Validator::JSONParserError)
+    end
+  end
+
+  describe 'When reading objects' do
+    it "recognizes empty objects" do
+      v = load_string '{}'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:object)).to be 1
+      expect(v.stats(:string)).to be 0
+    end
+
+    it "recognizes objects with a single attribute" do
+      v = load_string '{"key": "value"}'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:object)).to be 1
+      expect(v.stats(:string)).to be 2
+    end
+
+    it "recognizes objects with attributes of different types" do
+      v = load_string '{"k1": "value", "k2": -123.456, "k3": null}'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:object)).to be 1
+      expect(v.stats(:string)).to be 4
+      expect(v.stats(:literal)).to be 2
+    end
+
+    it "recognizes condensed objects (no whitespaces)" do
+      v = load_string '{"a":"b","c":"d"}'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:object)).to be 1
+      expect(v.stats(:string)).to be 4
+    end
+
+    it "recognizes formatted objects" do
+      v = load_string '{
+        "a":"b",
+        "c":"d"
+      }'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:object)).to be 1
+      expect(v.stats(:string)).to be 4
+    end
+
+    it "recognizes objects with nested objects and arrays" do
+      v = load_string '{
+        "a": {
+          "a1": "-",
+          "a2": "-",
+          "a3": {
+            "a3.1": "-"
+          },
+        },
+        "c": [1, null]
+      }'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:object)).to be 3
+      expect(v.stats(:array)).to be 1
+      expect(v.stats(:string)).to be 9
+      expect(v.stats(:literal)).to be 2
+    end
+
+    it "rejects objects without double-quoted attribute names" do
+      expect do
+        v = load_string '{a:"b",c:"d"}'
+        v.validate
+        end.to raise_error(FormatParser::JSONParser::Validator::JSONParserError)
+    end
+
+    it "rejects objects without comma separators" do
+      expect do
+        v = load_string '{
+          "a":"b"
+          "c":"d"
+        }'
+        v.validate
+        end.to raise_error(FormatParser::JSONParser::Validator::JSONParserError)
+    end
+  end
+
+  describe 'When reading arrays' do
+    it "recognizes empty arrays" do
+      v = load_string '[]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:array)).to be 1
+      expect(v.stats(:string)).to be 0
+    end
+
+    it "recognizes arrays with a single element" do
+      v = load_string '[{}]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:array)).to be 1
+      expect(v.stats(:object)).to be 1
+    end
+
+    it "recognizes arrays with elements of different types" do
+      v = load_string '[{"k1": "value"}, [], "a string", null, -123.456]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:array)).to be 2
+      expect(v.stats(:object)).to be 1
+      expect(v.stats(:string)).to be 3
+      expect(v.stats(:literal)).to be 2
+    end
+
+    it "recognizes condensed arrays (no whitespaces)" do
+      v = load_string '["a",2,null,false]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:array)).to be 1
+      expect(v.stats(:string)).to be 1
+      expect(v.stats(:literal)).to be 3
+    end
+
+    it "recognizes formatted arrays" do
+      v = load_string '[
+        {
+          "a":"b"
+        },
+        {
+          "c":"d"
+        }
+      ]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:array)).to be 1
+      expect(v.stats(:object)).to be 2
+      expect(v.stats(:string)).to be 4
+    end
+
+    it "recognizes arrays with nested objects and arrays" do
+      v = load_string '[{
+          "a": {
+            "a1": "-",
+            "a2": "-",
+            "a3": {
+              "a3.1": "-"
+            },
+          },
+          "c": [1, null]
+        },
+        [{ "a": "b" }, { "c":"d" }]
+      ]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:array)).to be 3
+      expect(v.stats(:object)).to be 5
+      expect(v.stats(:string)).to be 13
+      expect(v.stats(:literal)).to be 2
+    end
+
+    it "rejects arrays without comma separators" do
+      expect do
+        v = load_string '[
+          "abc"
+          "def"
+        ]'
+        v.validate
+        end.to raise_error(FormatParser::JSONParser::Validator::JSONParserError)
+    end
+  end
+
+  describe 'When reading strings' do
+    it "recognizes regular strings" do
+      v = load_string '["abc", "def", "ghi"]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:string)).to be 3
+    end
+
+    it "recognizes strings containing excaped characters" do
+      v = load_string '["ab\"c", "6\\2=3"]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:string)).to be 2
+    end
+
+    it "recognizes strings containing UTF8 characters" do
+      v = load_string '["abc😃🐶👀", "😃2🐶3👀"]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:string)).to be 2
+    end
+
+    it "recognizes long strings containing UTF8 characters" do
+      v = load_string '["aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀"]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:string)).to be 1
+    end
+  end
+
+  describe 'When reading literals' do
+    it "recognizes numbers" do
+      v = load_string '[1, -2.4, 1.0E+2]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:literal)).to be 3
+    end
+
+    it "recognizes boolean values" do
+      v = load_string '[true, false]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:literal)).to be 2
+    end
+
+    it "recognizes 'true', 'false' and 'null'" do
+      v = load_string '[true, false, null]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:literal)).to be 3
+    end
+  end
+
+  describe 'When reading invalid JSON content' do
+    it "rejects truncated JSON content" do
+      expect do
+        v = load_string '[{
+          "a": ["abc","def"],
+          "b": 4'
+        v.validate
+        end.to raise_error(FormatParser::JSONParser::Validator::JSONParserError)
+    end
+  end
+
+  describe 'When reading large JSON files' do
+    it "Returns 'false' without throwing errors when the initial chunk of a file is a valid JSON" do
+        v = load_file 'long_file_valid.json'
+
+        completed = v.validate
+        expect(completed).to be false
+    end
+
+    it "Returns 'false' without throwing errors when for long non-formatted JSON files" do
+      v = load_file 'long_file_valid_non_formatted.json'
+
+      completed = v.validate
+      expect(completed).to be false
+    end
+
+    it "Returns 'false' without throwing errors when the initial chunk of a file is a valid JSON even if there's an issue later" do
+      v = load_file 'long_file_malformed.json'
+
+      completed = v.validate
+      expect(completed).to be false
+    end
+  end
+end

data/spec/parsers/json_parser_spec.rb

@@ -0,0 +1,118 @@
+require 'spec_helper'
+
+describe FormatParser::JSONParser do
+  MAX_READS = 100
+
+  def load_file(file_name)
+    io = File.open(Pathname.new(fixtures_dir).join('JSON').join(file_name), 'rb')
+    FormatParser::ReadLimiter.new(io, max_reads: MAX_READS)
+  end
+
+  def file_size(file_name)
+    File.size(Pathname.new(fixtures_dir).join('JSON').join(file_name))
+  end
+
+  describe 'When reading objects valid JSON files' do
+    it "identifies JSON files with objects as root nodes" do
+      io = load_file 'object.json'
+
+      parsed = subject.call(io)
+
+      expect(parsed).not_to be_nil
+      expect(parsed.nature).to eq(:text)
+      expect(parsed.format).to eq(:json)
+      expect(parsed.content_type).to eq('application/json')
+    end
+
+    it "identifies JSON files carrying arrays as root nodes" do
+      io = load_file 'array.json'
+
+      parsed = subject.call(io)
+
+      expect(parsed).not_to be_nil
+      expect(parsed.nature).to eq(:text)
+      expect(parsed.format).to eq(:json)
+      expect(parsed.content_type).to eq('application/json')
+    end
+
+    it "identifies formatted JSON files" do
+      io = load_file 'formatted_object_utf8.json'
+
+      parsed = subject.call(io)
+
+      expect(parsed).not_to be_nil
+      expect(parsed.nature).to eq(:text)
+      expect(parsed.format).to eq(:json)
+      expect(parsed.content_type).to eq('application/json')
+    end
+
+    it "identifies files wrapped in whitespace characters" do
+      io = load_file 'whitespaces.json'
+
+      parsed = subject.call(io)
+
+      expect(parsed).not_to be_nil
+      expect(parsed.nature).to eq(:text)
+      expect(parsed.format).to eq(:json)
+      expect(parsed.content_type).to eq('application/json')
+    end
+
+    it "identifies files with nested objects and arrays" do
+      io = load_file 'nested_objects.json'
+
+      parsed = subject.call(io)
+
+      expect(parsed).not_to be_nil
+      expect(parsed.nature).to eq(:text)
+      expect(parsed.format).to eq(:json)
+      expect(parsed.content_type).to eq('application/json')
+    end
+
+    it "is reads the whole content of small files before accepting them" do
+      file_name = 'nested_objects.json'
+      io = load_file file_name
+      file_size = file_size file_name
+
+      parsed = subject.call(io)
+
+      expect(parsed).not_to be_nil
+      expect(parsed.nature).to eq(:text)
+      expect(parsed.format).to eq(:json)
+      expect(parsed.content_type).to eq('application/json')
+      expect(io.bytes).to be >= file_size
+    end
+
+    it "is accepts long files before reading the whole content" do
+      file_name = 'long_array_numbers.json'
+      io = load_file file_name
+      file_size = file_size file_name
+
+      parsed = subject.call(io)
+
+      expect(parsed).not_to be_nil
+      expect(parsed.nature).to eq(:text)
+      expect(parsed.format).to eq(:json)
+      expect(parsed.content_type).to eq('application/json')
+      expect(io.bytes).to be < file_size
+    end
+  end
+
+  describe 'When reading objects invalid JSON files' do
+    it "rejects files with corrupted JSON data" do
+      io = load_file 'invalid_malformed.json'
+
+      parsed = subject.call(io)
+
+      expect(parsed).to be_nil
+    end
+
+    it "rejects invalid files early without reading the whole content" do
+      io = load_file 'invalid_lorem_ipsum.json'
+
+      parsed = subject.call(io)
+
+      expect(parsed).to be_nil
+      expect(io.reads).to eq(1)
+    end
+  end
+end

data/spec/parsers/m3u_parser_spec.rb

@@ -11,7 +11,7 @@ describe FormatParser::M3UParser do
   end
 
   describe 'an m3u file with missing header' do
-    let(:m3u_file) { 'plain_text.m3u' }
+    let(:m3u_file) { 'invalid_plain_text.m3u' }
 
     it 'does not parse the file successfully' do
       expect(parsed_m3u).to be_nil

data/spec/parsers/mp3_parser_spec.rb

@@ -36,6 +36,12 @@ describe FormatParser::MP3Parser do
     expect(parsed).to be_nil
   end
 
+  it 'does not misdetect a WAV' do
+    fpath = fixtures_dir + '/WAV/c_SCAM_MIC_SOL001_RUN001.wav'
+    parsed = subject.call(File.open(fpath, 'rb'))
+    expect(parsed).to be_nil
+  end
+
   describe 'title/artist/album attributes' do
     let(:parsed) { subject.call(File.open(fpath, 'rb')) }
 

data/spec/parsers/ogg_parser_spec.rb

@@ -13,7 +13,7 @@ describe FormatParser::OggParser do
   end
 
   it 'skips a file if it contains more than MAX_POSSIBLE_OGG_PAGE_SIZE bytes of garbage at the end' do
-    parse_result = subject.call(File.open(__dir__ + '/../fixtures/Ogg/with_garbage_at_the_end.ogg', 'rb'))
+    parse_result = subject.call(File.open(__dir__ + '/../fixtures/Ogg/invalid_with_garbage_at_the_end.ogg', 'rb'))
     expect(parse_result).to be_nil
   end
 

data/spec/parsers/pdf_parser_spec.rb

@@ -46,17 +46,17 @@ describe FormatParser::PDFParser do
 
   describe 'broken PDF files should not parse' do
     it 'PDF with missing version header' do
-      parsed_pdf = parse_pdf 'not_a.pdf'
+      parsed_pdf = parse_pdf 'invalid_not_a.pdf'
       expect(parsed_pdf).to be_nil
     end
 
     it 'PDF 2.0 with offset start' do
-      parsed_pdf = parse_pdf 'PDF 2.0 with offset start.pdf'
+      parsed_pdf = parse_pdf 'invalid PDF 2.0 with offset start.pdf'
       expect(parsed_pdf).to be_nil
     end
 
     it 'exceeds the PDF read limit' do
-      parsed_pdf = parse_pdf 'exceed_PDF_read_limit.pdf'
+      parsed_pdf = parse_pdf 'invalid_exceed_PDF_read_limit.pdf'
       expect(parsed_pdf).to be_nil
     end
   end

data/spec/parsers/wav_parser_spec.rb

@@ -48,7 +48,7 @@ describe FormatParser::WAVParser do
 
   it "cannot parse file with audio format different from 1 and no 'fact' chunk" do
     expect {
-      subject.call(File.open(__dir__ + '/../fixtures/WAV/d_6_Channel_ID.wav', 'rb'))
+      subject.call(File.open(__dir__ + '/../fixtures/WAV/invalid_d_6_Channel_ID.wav', 'rb'))
     }.to raise_error(FormatParser::IOUtils::InvalidRead)
   end
 end

data/spec/parsers/webp_parser_spec.rb

@@ -7,7 +7,7 @@ describe FormatParser::WebpParser do
   end
 
   it 'does not parse files with an unrecognised variant' do
-    result = subject.call(File.open(fixtures_dir + 'WEBP/unrecognised-variant.webp', 'rb'))
+    result = subject.call(File.open(fixtures_dir + 'WEBP/invalid-unrecognised-variant.webp', 'rb'))
     expect(result).to be_nil
   end
 

data/spec/remote_fetching_spec.rb

@@ -104,6 +104,43 @@ describe 'Fetching data from HTTP remotes' do
     expect(file_information.format).to eq(:png)
   end
 
+  describe 'correctly parses WAV files without falling back to another filetype' do
+    ['c_8kmp316.wav', 'c_SCAM_MIC_SOL001_RUN001.wav'].each do |filename|
+      it "parses WAV file #{filename}" do
+        remote_url = 'http://localhost:9399/WAV/' + filename
+        file_information = FormatParser.parse_http(remote_url)
+        expect(file_information).not_to be_nil
+        expect(file_information.format).to eq(:wav)
+      end
+    end
+  end
+
+  describe "correctly parses files over HTTP without filename hint" do
+    Dir.glob(fixtures_dir + '/**/*.*').sort.each do |fixture_path|
+      file_name = File.basename(fixture_path)
+      next if file_name.include? "invalid"
+
+      file_type_dir = fixture_path.delete_prefix(fixtures_dir).delete_suffix(file_name)
+      file_type_dir.delete_prefix!('/').delete_suffix!('/')
+      next if file_type_dir.empty?
+
+      # skipping this one because it's a special case
+      next if file_name == "arch_many_entries.zip"
+
+      it "parses #{file_type_dir} file: #{file_name}" do
+        url = "http://localhost:9399/#{file_type_dir}/#{file_name}?some_param=test".gsub(' ', '%20')
+        result_with_hint = FormatParser.parse_http(url, filename_hint: file_name)
+        result_no_hint = FormatParser.parse_http(url)
+
+        expect(result_with_hint).not_to be_nil
+        expect(result_no_hint).not_to be_nil
+
+        expect(result_no_hint.nature).to eq(result_with_hint.nature)
+        expect(result_no_hint.format).to eq(result_with_hint.format)
+      end
+    end
+  end
+
   describe 'when parsing remote fixtures' do
     Dir.glob(fixtures_dir + '/**/*.*').sort.each do |fixture_path|
       filename = File.basename(fixture_path)

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 2.6.0
+  version: 2.7.1
 platform: ruby
 authors:
 - Noah Berman
 - Julik Tarkhanov
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-05-31 00:00:00.000000000 Z
+date: 2023-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: exifr
@@ -236,6 +236,8 @@ files:
 - lib/parsers/iso_base_media_file_format/decoder.rb
 - lib/parsers/iso_base_media_file_format/utils.rb
 - lib/parsers/jpeg_parser.rb
+- lib/parsers/json_parser.rb
+- lib/parsers/json_parser/validator.rb
 - lib/parsers/m3u_parser.rb
 - lib/parsers/mov_parser.rb
 - lib/parsers/mov_parser/decoder.rb
@@ -260,6 +262,7 @@ files:
 - lib/remote_io.rb
 - lib/string.rb
 - lib/text.rb
+- lib/utf8_reader.rb
 - lib/video.rb
 - spec/active_storage/blob_io_spec.rb
 - spec/active_storage/rails_app_spec.rb
@@ -289,6 +292,8 @@ files:
 - spec/parsers/iso_base_media_file_format/decoder_spec.rb
 - spec/parsers/iso_base_media_file_format/utils_spec.rb
 - spec/parsers/jpeg_parser_spec.rb
+- spec/parsers/json_parser/validator_spec.rb
+- spec/parsers/json_parser_spec.rb
 - spec/parsers/m3u_parser_spec.rb
 - spec/parsers/mov_parser_spec.rb
 - spec/parsers/mp3_parser_spec.rb
@@ -314,7 +319,7 @@ licenses:
 - MIT (Hippocratic)
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -329,8 +334,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
-signing_key:
+rubygems_version: 3.1.6
+signing_key: 
 specification_version: 4
 summary: A library for efficient parsing of file metadata
 test_files: []