format_parser 2.5.0 → 2.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be6d2146ac8ada9870883998a9d4e93a92b8bb572668c424e28be798b27cc21c
-  data.tar.gz: 5ea4e991625c8b7e1d1414e970efbc9d6fbc4827c4bb9e14ad2140fd0cbd396f
+  metadata.gz: f7b8b37e26143e2ea941db88183475e70c90ab658adcdad362b32b457a7d19ac
+  data.tar.gz: 50758e107065e1a2ab4fbca7775359d928cd1a62942277a94fcbabfefa1bfe10
 SHA512:
-  metadata.gz: '079c5c591cfd548ff4df342882048a0ad315c30256fd4dff9333c2944986cec7949ff85472658ef3a5156e4f1553964368257b838e1130664346f486ffde1623'
-  data.tar.gz: a877334ed908a65e03f91dfd83a4b0dc5ff5c569942335038c6b37a6727add95796dc023a7cdac1965ff8a3adfe0b131aefac3d3b30a2c33c4f565e3a60dcbc9
+  metadata.gz: d34bcd7b0162fe6f911bdd8c3b626dd9ce35b98139bca6ec1b54e653bc7e50453af734c74134745a46e91fbc3528d88f1663fabdbd35e06ae908a41f8e81dcd7
+  data.tar.gz: 2ebbc65f373a3e34a2e8300d40bb239df6d2003dab54b9c461a8cc75f651800d93273aa73bfb4f1a14f195e65f348c218b90b4c1bae860c8269e024ebb2e890c

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 2.6.0
+* Prevent the default loading of thumbnails on TIFF-based formats to improve I/O.
+
 ## 2.5.0
 * Add `avc1` and `xavc` as brand codes in the mp4 format parser to allow more file types to be parsed correctly.
 

data/README.md

@@ -26,6 +26,7 @@ and [dimensions,](https://github.com/sstephenson/dimensions) borrowing from them
 * HEIC
 * HEIF
 * JPEG
+* JSON
 * M3U
 * M4A
 * M4B

data/format_parser.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'exifr', '>= 1.3.8'
+  spec.add_dependency 'exifr', '>= 1.4.0'
   spec.add_dependency 'id3tag', '>= 0.14.2'
   spec.add_dependency 'matrix'
   spec.add_dependency 'measurometer'

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '2.5.0'
+  VERSION = '2.7.0'
 end

data/lib/format_parser.rb

@@ -17,6 +17,7 @@ module FormatParser
   require_relative 'read_limits_config'
   require_relative 'remote_io'
   require_relative 'io_constraint'
+  require_relative 'utf8_reader'
   require_relative 'care'
   require_relative 'active_storage/blob_analyzer'
   require_relative 'text'

data/lib/parsers/exif_parser.rb

@@ -175,7 +175,7 @@ module FormatParser::EXIFParser
   def exif_from_tiff_io(constrained_io, should_include_sub_ifds = false)
     Measurometer.instrument('format_parser.exif_parser.exif_from_tiff_io') do
       extended_io = IOExt.new(constrained_io)
-      exif_raw_data = EXIFR::TIFF.new(extended_io)
+      exif_raw_data = EXIFR::TIFF.new(extended_io, load_thumbnails: false)
 
       return unless exif_raw_data
 

data/lib/parsers/json_parser/validator.rb

@@ -0,0 +1,319 @@
+##
+# This class checks whether a given file is a valid JSON file.
+# The validation process DOES NOT assemble an object with the contents of the JSON file in memory,
+# Instead, it implements a simple state-machine-like that digests the contents of the file while traversing
+# the hierarchy of nodes in the document.
+#
+# Although this is based on the IETF standard (https://www.rfc-editor.org/rfc/rfc8259),
+# it does cut a few corners for the sake of simplicity. For instance, instead of validating
+# Numbers, "true", "false" and "null" tokens, it supports a type called Literal to hold generic sequences of characters.
+# This decision makes the implementation simpler while being a good-enough approach to identify JSON files.
+#
+# There is also a cap. Large files are not read all the way through. Instead, if the beginning of file is
+# JSON-compliant, it is assumed that the file is a JSON file.
+
+class FormatParser::JSONParser::Validator
+  class JSONParserError < StandardError
+  end
+
+  MAX_SAMPLE_SIZE = 1024
+  MAX_LITERAL_SIZE = 30 # much larger then necessary.
+  ESCAPE_CHAR = "\\"
+  WHITESPACE_CHARS = [" ", "\t", "\n", "\r"]
+  ENDING_VALUE_CHARS = [",", "]", "}"]
+  LITERALS_CHAR_TEMPLATE = /\w|[+\-.]/ # any alphanumeric, "+", "-" and "."
+
+  def initialize(io)
+    @io = io
+    @current_node = nil # :object, :array, :string, :literal
+    @parent_nodes = []
+    @current_state = :awaiting_root_node
+    @escape_next = false
+    @current_literal_size = 0
+    @pos = 0
+
+    @all_parsers = {}
+
+    @execution_stats = {
+      array: 0,
+      object: 0,
+      literal: 0,
+      string: 0
+    }
+
+    setup_transitions
+  end
+
+  def validate
+    char_reader = FormatParser::UTF8Reader.new(@io)
+
+    while (c = char_reader.read_char)
+      @pos += 1
+      parse_char c
+
+      # Halt validation if the sampling limit is reached.
+      if @pos >= MAX_SAMPLE_SIZE
+        raise JSONParserError, "Invalid JSON file" if @current_state == :awaiting_root_node
+        return false
+      end
+    end
+
+    # Raising error in case the EOF is reached earlier than expected
+    raise JSONParserError, "Incomplete JSON file" if @current_state != :closed
+    true
+  rescue  FormatParser::UTF8Reader::UTF8CharReaderError
+    raise JSONParserError, "Invalid UTF-8 character"
+  end
+
+  def stats(node_type)
+    @execution_stats[node_type]
+  end
+
+  private
+
+  def setup_transitions
+    when_its :awaiting_root_node, ->(c) do
+      read_whitespace(c) or
+        start_object(c) or
+        start_array(c)
+    end
+
+    when_its :awaiting_object_attribute_key, ->(c) do
+      read_whitespace(c) or
+        start_attribute_key(c) or
+        close_object(c)
+    end
+
+    when_its :reading_object_attribute_key, ->(c) do
+      close_attribute_key(c) or
+        read_valid_string_char(c)
+    end
+
+    when_its :awaiting_object_colon_separator, ->(c) do
+      read_whitespace(c) or
+        read_colon(c)
+    end
+
+    when_its :awaiting_object_attribute_value, ->(c) do
+      read_whitespace(c) or
+        start_object(c) or
+        start_array(c) or
+        start_string(c) or
+        start_literal(c)
+    end
+
+    when_its :awaiting_array_value, ->(c) do
+      read_whitespace(c) or
+        start_object(c) or
+        start_array(c) or
+        start_string(c) or
+        start_literal(c) or
+        close_array(c)
+    end
+
+    when_its :reading_string, ->(c) do
+      close_string(c) or
+        read_valid_string_char(c)
+    end
+
+    when_its :awaiting_next_or_close, ->(c) do
+      read_whitespace(c) or
+        read_comma_separator(c) or
+        close_object(c) or
+        close_array(c)
+    end
+
+    when_its :reading_literal, ->(c) do
+      read_valid_literal_char(c) or (
+        close_literal(c) and (
+          read_whitespace(c) or
+          read_comma_separator(c) or
+          close_array(c) or
+          close_object(c)))
+    end
+
+    when_its :closed, ->(c) do
+      read_whitespace(c)
+    end
+  end
+
+  def when_its(state, act)
+    @all_parsers[state] = act
+  end
+
+  def parse_char(c)
+    next_step = @all_parsers[@current_state]
+    accepted = next_step.call(c)
+    reject_char(c) unless accepted
+  end
+
+  def read_whitespace(c)
+    whitespace?(c)
+  end
+
+  def read_colon(c)
+    if c == ":"
+      @current_state = :awaiting_object_attribute_value
+      return true
+    end
+    false
+  end
+
+  def read_valid_string_char(c)
+    if @escape_next
+      @escape_next = false
+      return true
+    end
+
+    if c == ESCAPE_CHAR
+      @escape_next = true
+      return true
+    end
+    !control_char?(c) and c != "\""
+  end
+
+  def read_valid_literal_char(c)
+    if valid_literal_char?(c)
+      @current_literal_size += 1
+      return true
+    end
+
+    false
+  end
+
+  def read_comma_separator(c)
+    if c == ","
+      @current_state = :awaiting_object_attribute_key if @current_node == :object
+      @current_state = :awaiting_array_value if @current_node == :array
+      return true
+    end
+    false
+  end
+
+  # Object: {"k1":"val", "k2":[1,2,3], "k4": undefined, "k5": {"l1": 6}}
+  def start_object(c)
+    return false if whitespace?(c)
+    return false unless c == "{"
+
+    begin_node(:object)
+    @current_state = :awaiting_object_attribute_key
+    true
+  end
+
+  def close_object(c)
+    return false if whitespace?(c)
+    return false unless @current_node == :object and c == "}"
+
+    end_node
+    @current_state = :awaiting_next_or_close unless @current_node.nil?
+    true
+  end
+
+  # Array: [1, "two", true, undefined, {}, []]
+  def start_array(c)
+    return false unless c == "["
+
+    begin_node(:array)
+    @current_state = :awaiting_array_value
+    true
+  end
+
+  def close_array(c)
+    return false if whitespace?(c)
+    return false unless @current_node == :array and c == "]"
+
+    end_node
+    @current_state = :awaiting_next_or_close unless @current_node.nil?
+    true
+  end
+
+  def start_attribute_key(c)
+    return false unless c == "\""
+
+    begin_node(:string)
+    @current_state = :reading_object_attribute_key
+    true
+  end
+
+  def close_attribute_key(c)
+    return false if @escape_next
+    return false unless c == "\""
+    end_node
+    @current_state = :awaiting_object_colon_separator
+    true
+  end
+
+  # Strings: "Foo"
+  def start_string(c)
+    return false unless c == "\""
+
+    begin_node(:string)
+    @current_state = :reading_string
+    true
+  end
+
+  def close_string(c)
+    return false if @escape_next
+    return false unless c == "\""
+    end_node
+    @current_state = :awaiting_next_or_close
+    true
+  end
+
+  # literals: null, undefined, true, false, NaN, infinity, -123.456e10 -123,456e10
+  def start_literal(c)
+    return false unless valid_literal_char?(c)
+
+    begin_node(:literal)
+    @current_state = :reading_literal
+    @current_literal_size = 1
+    true
+  end
+
+  def close_literal(c)
+    raise JSONParserError, "Literal to large at #{@pos}" if @current_literal_size > MAX_LITERAL_SIZE
+
+    if whitespace?(c) || ENDING_VALUE_CHARS.include?(c)
+      end_node
+      @current_state = :awaiting_next_or_close
+      return true
+    end
+
+    false
+  end
+
+  # Marks the creation of a node (object, array, string or literal)
+  def begin_node(node_type)
+    # Accounting for the new node
+    @execution_stats[node_type] ||= 0
+    @execution_stats[node_type] += 1
+
+    # Managing the node execution stack
+    @parent_nodes.push(@current_node)
+    @current_node = node_type
+  end
+
+  # Marks the closure of a node (object, array, string or literal)
+  def end_node
+    @current_node = @parent_nodes.pop
+    @current_state = :closed if @current_node.nil?
+  end
+
+  def reject_char(char)
+    raise JSONParserError, "Unexpected char #{char} in position #{@pos}"
+  end
+
+  def whitespace?(c)
+    WHITESPACE_CHARS.include?(c)
+  end
+
+  def control_char?(c)
+    # control characters: (U+0000 through U+001F)
+    utf8_code = c.unpack('U*')[0]
+    utf8_code <= 31
+  end
+
+  def valid_literal_char?(c)
+    LITERALS_CHAR_TEMPLATE === c
+  end
+end

data/lib/parsers/json_parser.rb

@@ -0,0 +1,25 @@
+class FormatParser::JSONParser
+  include FormatParser::IOUtils
+  require_relative 'json_parser/validator'
+
+  JSON_MIME_TYPE = 'application/json'
+
+  def likely_match?(filename)
+    filename =~ /\.json$/i
+  end
+
+  def call(io)
+    io = FormatParser::IOConstraint.new(io)
+    validator = Validator.new(io)
+
+    validator.validate
+
+    FormatParser::Text.new(
+      format: :json,
+      content_type: JSON_MIME_TYPE,
+    )
+  rescue Validator::JSONParserError
+    nil
+  end
+  FormatParser.register_parser new, natures: :text, formats: :json
+end

data/lib/utf8_reader.rb

@@ -0,0 +1,68 @@
+##
+# This class Reads individual characters from files using UTF-8 encoding
+# This deals with two main concerns:
+#   - Variable byte length of characters
+#   - Reducing the number of read operations by loading bytes in chunks
+
+class FormatParser::UTF8Reader
+  READ_CHUNK_SIZE = 128
+
+  class UTF8CharReaderError < StandardError
+  end
+
+  def initialize(io)
+    @io = io
+    @chunk = ""
+    @index = 0
+    @eof = false
+  end
+
+  def read_char
+    first_byte = read_byte
+    return if first_byte.nil?
+
+    char_length = assess_char_length(first_byte)
+    as_bytes = Array.new(char_length) do |i|
+      next first_byte if i == 0
+      read_byte
+    end
+
+    char = as_bytes.pack('c*').force_encoding('UTF-8')
+    raise UTF8CharReaderError, "Invalid UTF-8 character" unless char.valid_encoding?
+
+    char
+  rescue TypeError
+    raise UTF8CharReaderError, "Invalid UTF-8 character"
+  end
+
+  private
+
+  def read_byte
+    manage_data_chunk
+    return if @chunk.nil?
+    byte = @chunk.bytes[@index]
+    @index += 1 unless byte.nil?
+    byte
+  end
+
+  def manage_data_chunk
+    return if @index < @chunk.length
+    @chunk = @io.read(READ_CHUNK_SIZE)
+    @chunk ||= ""
+    @index = 0
+    @eof = true if @chunk.nil? or @chunk.length < READ_CHUNK_SIZE
+  end
+
+  def assess_char_length(first_byte)
+    # 0_______ (1 byte)
+    # 110_____ (2 bytes) 192
+    # 1110____ (3 bytes) 224
+    # 11110___ (4 bytes) 240
+    case first_byte
+    when 240.. then 4
+    when 224..239 then 3
+    when 192..223 then 2
+    else 1
+    end
+  end
+end

data/spec/parsers/json_parser/validator_spec.rb

@@ -0,0 +1,321 @@
+require 'spec_helper'
+
+describe FormatParser::JSONParser::Validator do
+  def load_file(file_name)
+    io = File.open(Pathname.new(fixtures_dir).join('JSON').join(file_name), 'rb')
+    FormatParser::JSONParser::Validator.new(io)
+  end
+
+  def load_string(content)
+    io = StringIO.new(content.encode(Encoding::UTF_8))
+    FormatParser::JSONParser::Validator.new(io)
+  end
+
+  describe 'When reading root nodes' do
+    it "identifies objects as root nodes" do
+      v = load_string '{"key": "value"}'
+
+      completed = v.validate
+
+      expect(completed).to be true
+      expect(v.stats(:object)).to be 1
+      expect(v.stats(:string)).to be 2
+    end
+
+    it "identifies arrays as root nodes" do
+      v = load_string '["e1", "e2"]'
+
+      completed = v.validate
+
+      expect(completed).to be true
+      expect(v.stats(:array)).to be 1
+      expect(v.stats(:string)).to be 2
+    end
+
+    it "rejects strings as root nodes" do
+      expect do
+        v = load_string '"this is a string"'
+        v.validate
+        end.to raise_error(FormatParser::JSONParser::Validator::JSONParserError)
+    end
+
+    it "rejects literals as root nodes" do
+      expect do
+        v = load_string 'true'
+        v.validate
+        end.to raise_error(FormatParser::JSONParser::Validator::JSONParserError)
+    end
+  end
+
+  describe 'When reading objects' do
+    it "recognizes empty objects" do
+      v = load_string '{}'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:object)).to be 1
+      expect(v.stats(:string)).to be 0
+    end
+
+    it "recognizes objects with a single attribute" do
+      v = load_string '{"key": "value"}'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:object)).to be 1
+      expect(v.stats(:string)).to be 2
+    end
+
+    it "recognizes objects with attributes of different types" do
+      v = load_string '{"k1": "value", "k2": -123.456, "k3": null}'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:object)).to be 1
+      expect(v.stats(:string)).to be 4
+      expect(v.stats(:literal)).to be 2
+    end
+
+    it "recognizes condensed objects (no whitespaces)" do
+      v = load_string '{"a":"b","c":"d"}'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:object)).to be 1
+      expect(v.stats(:string)).to be 4
+    end
+
+    it "recognizes formatted objects" do
+      v = load_string '{
+        "a":"b",
+        "c":"d"
+      }'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:object)).to be 1
+      expect(v.stats(:string)).to be 4
+    end
+
+    it "recognizes objects with nested objects and arrays" do
+      v = load_string '{
+        "a": {
+          "a1": "-",
+          "a2": "-",
+          "a3": {
+            "a3.1": "-"
+          },
+        },
+        "c": [1, null]
+      }'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:object)).to be 3
+      expect(v.stats(:array)).to be 1
+      expect(v.stats(:string)).to be 9
+      expect(v.stats(:literal)).to be 2
+    end
+
+    it "rejects objects without double-quoted attribute names" do
+      expect do
+        v = load_string '{a:"b",c:"d"}'
+        v.validate
+        end.to raise_error(FormatParser::JSONParser::Validator::JSONParserError)
+    end
+
+    it "rejects objects without comma separators" do
+      expect do
+        v = load_string '{
+          "a":"b"
+          "c":"d"
+        }'
+        v.validate
+        end.to raise_error(FormatParser::JSONParser::Validator::JSONParserError)
+    end
+  end
+
+  describe 'When reading arrays' do
+    it "recognizes empty arrays" do
+      v = load_string '[]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:array)).to be 1
+      expect(v.stats(:string)).to be 0
+    end
+
+    it "recognizes arrays with a single element" do
+      v = load_string '[{}]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:array)).to be 1
+      expect(v.stats(:object)).to be 1
+    end
+
+    it "recognizes arrays with elements of different types" do
+      v = load_string '[{"k1": "value"}, [], "a string", null, -123.456]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:array)).to be 2
+      expect(v.stats(:object)).to be 1
+      expect(v.stats(:string)).to be 3
+      expect(v.stats(:literal)).to be 2
+    end
+
+    it "recognizes condensed arrays (no whitespaces)" do
+      v = load_string '["a",2,null,false]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:array)).to be 1
+      expect(v.stats(:string)).to be 1
+      expect(v.stats(:literal)).to be 3
+    end
+
+    it "recognizes formatted arrays" do
+      v = load_string '[
+        {
+          "a":"b"
+        },
+        {
+          "c":"d"
+        }
+      ]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:array)).to be 1
+      expect(v.stats(:object)).to be 2
+      expect(v.stats(:string)).to be 4
+    end
+
+    it "recognizes arrays with nested objects and arrays" do
+      v = load_string '[{
+          "a": {
+            "a1": "-",
+            "a2": "-",
+            "a3": {
+              "a3.1": "-"
+            },
+          },
+          "c": [1, null]
+        },
+        [{ "a": "b" }, { "c":"d" }]
+      ]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:array)).to be 3
+      expect(v.stats(:object)).to be 5
+      expect(v.stats(:string)).to be 13
+      expect(v.stats(:literal)).to be 2
+    end
+
+    it "rejects arrays without comma separators" do
+      expect do
+        v = load_string '[
+          "abc"
+          "def"
+        ]'
+        v.validate
+        end.to raise_error(FormatParser::JSONParser::Validator::JSONParserError)
+    end
+  end
+
+  describe 'When reading strings' do
+    it "recognizes regular strings" do
+      v = load_string '["abc", "def", "ghi"]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:string)).to be 3
+    end
+
+    it "recognizes strings containing excaped characters" do
+      v = load_string '["ab\"c", "6\\2=3"]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:string)).to be 2
+    end
+
+    it "recognizes strings containing UTF8 characters" do
+      v = load_string '["abc😃🐶👀", "😃2🐶3👀"]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:string)).to be 2
+    end
+
+    it "recognizes long strings containing UTF8 characters" do
+      v = load_string '["aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀aØڃಚ😁🐶👀"]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:string)).to be 1
+    end
+  end
+
+  describe 'When reading literals' do
+    it "recognizes numbers" do
+      v = load_string '[1, -2.4, 1.0E+2]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:literal)).to be 3
+    end
+
+    it "recognizes boolean values" do
+      v = load_string '[true, false]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:literal)).to be 2
+    end
+
+    it "recognizes 'true', 'false' and 'null'" do
+      v = load_string '[true, false, null]'
+
+      completed = v.validate
+      expect(completed).to be true
+      expect(v.stats(:literal)).to be 3
+    end
+  end
+
+  describe 'When reading invalid JSON content' do
+    it "rejects truncated JSON content" do
+      expect do
+        v = load_string '[{
+          "a": ["abc","def"],
+          "b": 4'
+        v.validate
+        end.to raise_error(FormatParser::JSONParser::Validator::JSONParserError)
+    end
+  end
+
+  describe 'When reading large JSON files' do
+    it "Returns 'false' without throwing errors when the initial chunk of a file is a valid JSON" do
+        v = load_file 'long_file_valid.json'
+
+        completed = v.validate
+        expect(completed).to be false
+    end
+
+    it "Returns 'false' without throwing errors when for long non-formatted JSON files" do
+      v = load_file 'long_file_valid_non_formatted.json'
+
+      completed = v.validate
+      expect(completed).to be false
+    end
+
+    it "Returns 'false' without throwing errors when the initial chunk of a file is a valid JSON even if there's an issue later" do
+      v = load_file 'long_file_malformed.json'
+
+      completed = v.validate
+      expect(completed).to be false
+    end
+  end
+end

data/spec/parsers/json_parser_spec.rb

@@ -0,0 +1,118 @@
+require 'spec_helper'
+
+describe FormatParser::JSONParser do
+  MAX_READS = 100
+
+  def load_file(file_name)
+    io = File.open(Pathname.new(fixtures_dir).join('JSON').join(file_name), 'rb')
+    FormatParser::ReadLimiter.new(io, max_reads: MAX_READS)
+  end
+
+  def file_size(file_name)
+    File.size(Pathname.new(fixtures_dir).join('JSON').join(file_name))
+  end
+
+  describe 'When reading objects valid JSON files' do
+    it "identifies JSON files with objects as root nodes" do
+      io = load_file 'object.json'
+
+      parsed = subject.call(io)
+
+      expect(parsed).not_to be_nil
+      expect(parsed.nature).to eq(:text)
+      expect(parsed.format).to eq(:json)
+      expect(parsed.content_type).to eq('application/json')
+    end
+
+    it "identifies JSON files carrying arrays as root nodes" do
+      io = load_file 'array.json'
+
+      parsed = subject.call(io)
+
+      expect(parsed).not_to be_nil
+      expect(parsed.nature).to eq(:text)
+      expect(parsed.format).to eq(:json)
+      expect(parsed.content_type).to eq('application/json')
+    end
+
+    it "identifies formatted JSON files" do
+      io = load_file 'formatted_object_utf8.json'
+
+      parsed = subject.call(io)
+
+      expect(parsed).not_to be_nil
+      expect(parsed.nature).to eq(:text)
+      expect(parsed.format).to eq(:json)
+      expect(parsed.content_type).to eq('application/json')
+    end
+
+    it "identifies files wrapped in whitespace characters" do
+      io = load_file 'whitespaces.json'
+
+      parsed = subject.call(io)
+
+      expect(parsed).not_to be_nil
+      expect(parsed.nature).to eq(:text)
+      expect(parsed.format).to eq(:json)
+      expect(parsed.content_type).to eq('application/json')
+    end
+
+    it "identifies files with nested objects and arrays" do
+      io = load_file 'nested_objects.json'
+
+      parsed = subject.call(io)
+
+      expect(parsed).not_to be_nil
+      expect(parsed.nature).to eq(:text)
+      expect(parsed.format).to eq(:json)
+      expect(parsed.content_type).to eq('application/json')
+    end
+
+    it "is reads the whole content of small files before accepting them" do
+      file_name = 'nested_objects.json'
+      io = load_file file_name
+      file_size = file_size file_name
+
+      parsed = subject.call(io)
+
+      expect(parsed).not_to be_nil
+      expect(parsed.nature).to eq(:text)
+      expect(parsed.format).to eq(:json)
+      expect(parsed.content_type).to eq('application/json')
+      expect(io.bytes).to be >= file_size
+    end
+
+    it "is accepts long files before reading the whole content" do
+      file_name = 'long_array_numbers.json'
+      io = load_file file_name
+      file_size = file_size file_name
+
+      parsed = subject.call(io)
+
+      expect(parsed).not_to be_nil
+      expect(parsed.nature).to eq(:text)
+      expect(parsed.format).to eq(:json)
+      expect(parsed.content_type).to eq('application/json')
+      expect(io.bytes).to be < file_size
+    end
+  end
+
+  describe 'When reading objects invalid JSON files' do
+    it "rejects files with corrupted JSON data" do
+      io = load_file 'malformed.json'
+
+      parsed = subject.call(io)
+
+      expect(parsed).to be_nil
+    end
+
+    it "rejects invalid files early without reading the whole content" do
+      io = load_file 'lorem_ipsum.json'
+
+      parsed = subject.call(io)
+
+      expect(parsed).to be_nil
+      expect(io.reads).to eq(1)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 2.5.0
+  version: 2.7.0
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-12 00:00:00.000000000 Z
+date: 2023-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: exifr
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.8
+        version: 1.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.8
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: id3tag
   requirement: !ruby/object:Gem::Requirement
@@ -236,6 +236,8 @@ files:
 - lib/parsers/iso_base_media_file_format/decoder.rb
 - lib/parsers/iso_base_media_file_format/utils.rb
 - lib/parsers/jpeg_parser.rb
+- lib/parsers/json_parser.rb
+- lib/parsers/json_parser/validator.rb
 - lib/parsers/m3u_parser.rb
 - lib/parsers/mov_parser.rb
 - lib/parsers/mov_parser/decoder.rb
@@ -260,6 +262,7 @@ files:
 - lib/remote_io.rb
 - lib/string.rb
 - lib/text.rb
+- lib/utf8_reader.rb
 - lib/video.rb
 - spec/active_storage/blob_io_spec.rb
 - spec/active_storage/rails_app_spec.rb
@@ -289,6 +292,8 @@ files:
 - spec/parsers/iso_base_media_file_format/decoder_spec.rb
 - spec/parsers/iso_base_media_file_format/utils_spec.rb
 - spec/parsers/jpeg_parser_spec.rb
+- spec/parsers/json_parser/validator_spec.rb
+- spec/parsers/json_parser_spec.rb
 - spec/parsers/m3u_parser_spec.rb
 - spec/parsers/mov_parser_spec.rb
 - spec/parsers/mp3_parser_spec.rb
@@ -329,7 +334,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: A library for efficient parsing of file metadata