format_parser 0.12.2 → 0.12.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7dce2d0bca62e45c867866920dabc1b302f3ec3823e9a2602b1898699213694f
-  data.tar.gz: 8e4e2826467614c21d687648123cae1326976bcc4f6f344a40d35a3bac16f988
+  metadata.gz: 03325dd0dc412571fd66d0c94226b9b88d7ae88b9bb6a985fe7c29f226c87b64
+  data.tar.gz: cbb1e33a3bffa36e832a064d9f28c6f8ebb2985fee397560d3ce19d971c36b96
 SHA512:
-  metadata.gz: dbe1cf517017ee9d0ec8ada4cde56f0a61dac1ae638d1e0059f5f717b0230b9fb6eb521a89e37fddfc3f441b2b457d534939acf0fa2d1dd0491d16a30a4975ce
-  data.tar.gz: f4c53aa7f9b8608c7878607081768342880254963de4ae5b0eeebe2d0dd2901a2d8acd6f260a3743eca0cd3fdabb825cba94884856a1824453e35c88e3f1f733
+  metadata.gz: 95c9d34a4469670ee69b718dac997fc3d1cc471071979ceb9df0010bff191d8b4c39e02f34669a13d831e40d918ad79889a6ff3b3ff0272df40b6160ee63bf8c
+  data.tar.gz: 417dbdcd4ea63939e060df8a8dd6e4927b4d9ec2de01cfab7d7b9ecf9701f56c5a93494cae7fb30361178d2b05b840f6c55928fffed73742829374acc8e24aff

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.12.4
+* Ensure JPEG recognition only runs when the JPEG SOI marker is detected **at the start** of file. Previously
+  the JPEG parser would scan for the marker, sometimes finding it (appropriately) in places like... MP3 album
+  artwork inside ID3 tags. Or Keynote documents. Or whatnot - lots of things have JPEG thumbnails embedded.
+
+## 0.12.3
+* Make sure all strings going to the JSON representations of parse results are encoded as UTF-8 or escaped
+
 ## 0.12.2
 * Make sure the `VERSION` constant is available in the loaded gem. Previously the constant would be made
   available by Bundler when developing the library - since it loads the `.gemspec` which, in turn, requires the

data/lib/attributes_json.rb

@@ -11,6 +11,8 @@
 #   the_foo.number_of_bars = 42
 #   the_foo.as_json #=> {:number_of_bars => 42}
 module FormatParser::AttributesJSON
+  UNICODE_REPLACEMENT_CHAR = [0xFFFD].pack('U')
+  MAXIMUM_JSON_NESTING_WHEN_SANITIZING = 256
 
   # Implements a sane default `as_json` for an object
   # that accessors defined
@@ -19,11 +21,12 @@ module FormatParser::AttributesJSON
     h['nature'] = nature if respond_to?(:nature) # Needed for file info structs
     methods.grep(/\w\=$/).each_with_object(h) do |attr_writer_method_name, h|
       reader_method_name = attr_writer_method_name.to_s.gsub(/\=$/, '')
-      value = public_send(reader_method_name)
-      value = nil if value == Float::INFINITY
-      # When calling as_json on our members there is no need to pass the root: option given to us
-      # by the caller
-      h[reader_method_name] = value.respond_to?(:as_json) ? value.as_json : value
+      attribute_value = public_send(reader_method_name)
+      # When calling as_json on our members there is no need to pass
+      # the root: option given to us by the caller
+      unwrapped_attribute_value = attribute_value.respond_to?(:as_json) ? attribute_value.as_json : attribute_value
+      sanitized_value = _sanitize_json_value(unwrapped_attribute_value)
+      h[reader_method_name] = sanitized_value
     end
     if root
       {'format_parser_file_info' => h}
@@ -32,6 +35,32 @@ module FormatParser::AttributesJSON
     end
   end
 
+  # Used for sanitizing values that are sourced to `JSON::Generator::State#generate`
+  # The reason we need to do this is as follows: `JSON.generate / JSON.dump / JSON.pretty_generate`
+  # use a totally different code path than `"foo".to_json(generator_state)`. We cannot predict
+  # which one of these two ways our users will be using, and at the same time we need to prevent
+  # invalid Strings (ones which cannot be encoded into UTF-8) as well as Float::INFINITY values
+  # from being passed to the JSON encoder. Since we cannot override the JSON generator with
+  # these additions, instead we will deep-convert the entire object being output to make sure
+  # it is up to snuff.
+  def _sanitize_json_value(value, nesting = 0)
+    raise ArgumentError, 'Nested JSON-ish structure too deep' if nesting > MAXIMUM_JSON_NESTING_WHEN_SANITIZING
+    case value
+    when Float::INFINITY
+      nil
+    when String
+      value.encode(Encoding::UTF_8, undef: :replace, replace: UNICODE_REPLACEMENT_CHAR)
+    when Hash
+      Hash[value.map { |k, v| [_sanitize_json_value(k, nesting + 1), _sanitize_json_value(v, nesting + 1)] }]
+    when Array
+      value.map { |v| _sanitize_json_value(v, nesting + 1) }
+    when Struct
+      _sanitize_json_value(value.to_h, nesting + 1)
+    else
+      value
+    end
+  end
+
   # Implements to_json with sane defaults, with or without arguments
   def to_json(*maybe_generator_state)
     as_json(root: false).to_json(*maybe_generator_state)

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.12.2'
+  VERSION = '0.12.4'
 end

data/lib/parsers/jpeg_parser.rb

@@ -5,7 +5,7 @@ class FormatParser::JPEGParser
   class InvalidStructure < StandardError
   end
 
-  SOI_MARKER = 0xD8 # start of image
+  JPEG_SOI_MARKER_HEAD = [0xFF, 0xD8].pack('C2')
   SOF_MARKERS = [0xC0..0xC3, 0xC5..0xC7, 0xC9..0xCB, 0xCD..0xCF]
   EOI_MARKER  = 0xD9  # end of image
   SOS_MARKER  = 0xDA  # start of stream
@@ -32,9 +32,13 @@ class FormatParser::JPEGParser
   end
 
   def scan
-    # Return early if it is not a JPEG at all
-    signature = read_next_marker
-    return unless signature == SOI_MARKER
+    # Most JPEG images start with the 0xFF0xD8 SOI marker.
+    # We _can_ search for that marker, but we will then
+    # ambiguously capture things like JPEGs embedded in ID3
+    # tags of MP3s - these _are_ JPEGs but we care much
+    # more about the top-level "wrapper" file, not about
+    # it's bits and bobs
+    return unless safe_read(@buf, 2) == JPEG_SOI_MARKER_HEAD
 
     markers_start_at = @buf.pos
 

data/spec/attributes_json_spec.rb

@@ -97,4 +97,47 @@ describe FormatParser::AttributesJSON do
 
     expect(readback).to have_key(:nature)
   end
+
+  it 'converts purely-binary String objects deeply nested in the struct to escapes and question marks' do
+    nasty_hash = {
+      id: 'TIT2',
+      size: 37,
+      flags: "\x00\x00",
+      struct: Struct.new(:key).new('Value'),
+      content: "\x01\xFF\xFEb\x00i\x00r\x00d\x00s\x00 \x005\x00 \x00m\x00o\x00r\x00e\x00 \x00c\x00o\x00m\x00p\x00".b
+    }
+    expect {
+      JSON.pretty_generate(nasty_hash) # Should not raise an error
+    }.to raise_error(Encoding::UndefinedConversionError)
+
+    anon_class = Struct.new(:evil)
+    anon_class.include FormatParser::AttributesJSON
+
+    object_with_attributes_module = anon_class.new(nasty_hash)
+    output = JSON.pretty_generate(object_with_attributes_module)
+
+    parsed_output = JSON.parse(output, symbolize_names: true)
+
+    expect(parsed_output[:evil][:struct]).to eq(key: 'Value')
+    expect(parsed_output[:evil][:id]).to eq('TIT2')
+    expect(parsed_output[:evil][:flags]).to be_kind_of(String)
+  end
+
+  it 'prevents traversals of data structures which are too deep with an exception' do
+    fractal_hash = {}
+    current = fractal_hash
+    1024.times do
+      current[:leaf] = {}
+      current = current[:leaf]
+    end
+
+    anon_class = Struct.new(:evil)
+    anon_class.include FormatParser::AttributesJSON
+
+    object_with_attributes_module = anon_class.new(fractal_hash)
+
+    expect {
+      JSON.pretty_generate(object_with_attributes_module)
+    }.to raise_error(/structure too deep/)
+  end
 end

data/spec/esoteric_formats_spec.rb

@@ -14,6 +14,13 @@ describe 'Parsing esoteric files and files causing ambiguous detection' do
     expect(result.nature).to eq(:archive)
   end
 
+  it 'does not pick up JPG album art within an MP3 as a JPEG file' do
+    jpeg_path = fixtures_dir + '/MP3/ATC Fixture With Album Art.mp3'
+    results = FormatParser.parse(File.open(jpeg_path, 'rb'), results: :all)
+    expect(results).to be_one
+    expect(results.first.nature).to eq(:audio)
+  end
+
   it 'returns a result for JPEG file that causes many reads due to too many APP1 markers' do
     jpeg_path = fixtures_dir + '/JPEG/too_many_APP1_markers_surrogate.jpg'
     result = FormatParser.parse(File.open(jpeg_path, 'rb'))

data/spec/parsers/zip_parser_spec.rb

@@ -82,7 +82,6 @@ describe FormatParser::ZIPParser do
     expect(json_parsed_repr[:format]).to eq('zip')
     expect(json_parsed_repr[:entries]).to be_kind_of(Array)
     expect(json_parsed_repr[:entries].length).to eq(3)
-
     json_parsed_repr[:entries].each do |e|
       expect(e[:filename]).to be_kind_of(String)
       expect(e[:size]).to be_kind_of(Integer)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.12.2
+  version: 0.12.4
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-05-06 00:00:00.000000000 Z
+date: 2018-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks