drjson 1.0.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in drjson.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Matthias Luedtke - http://github.com/mat
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,42 @@
+# Dr. JSON
+
+Closes abruptly cut-off JSON strings.
+
+## Installation
+
+Install the gem:
+
+    $ gem install drjson
+
+Or add this line to your application's Gemfile:
+
+    gem 'drjson'
+
+And then execute:
+
+    $ bundle
+
+## Usage
+
+You can hand over your poor JSON over command line
+
+    $ echo '{"foo":nul' | drjson 
+    {"foo":null}
+
+    $ echo -n '[7, [42' | drjson
+    [7, [42]]
+
+or via file
+
+    $ echo -n '{"foo": {"bar"' > my_file.json
+    $ drjson my_file.json
+    {"foo": {"bar":null}}
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/drjson

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require 'drjson'
+
+if filename = ARGV[0]
+  json_like = File.read(filename)
+else
+  json_like = $stdin.read
+end
+
+json = DrJson.new.repair(json_like)
+puts json
+

data/drjson.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'drjson/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "drjson"
+  gem.version       = Drjson::VERSION
+  gem.authors       = ["Matthias Luedtke"]
+  gem.email         = ["github@matthias.luedtke.de"]
+  gem.description   = %q{Closes abruptly cut-off JSON strings.}
+  gem.summary       = gem.description
+  gem.homepage      = "http://drjson.github.com/"
+
+  gem.files         = (`git ls-files`.split($/)).reject{ |path| path =~ /fixtures/}
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+end

data/lib/drjson.rb

@@ -0,0 +1,217 @@
+
+# DrJson closes abruptly cut-off JSON strings.
+#
+#     $ echo '{"foo":nul' | drjson 
+#     {"foo":null}
+# 
+# Regular, well-formed JSON is passed through untouched.
+#
+# Install with rubygems
+#
+#     $ gem install drjson
+#
+# Find the [source on GitHub][so].
+#
+# [so]: https://github.com/mat/drjson/
+
+require 'strscan'
+
+class DrJson
+
+  ### repair(json)
+
+  # Parses and repairs the possbily cut-off input `json_str`
+  # in recursive descent fashion. Returns `json_str` with 
+  # necesarry closing tags appended.
+  def repair(json_str)
+    @input = StringScanner.new(json_str)
+    @result = ""
+ 
+    begin
+      # Real world JSON has arrays as top level elements, yep.
+      parse_object || parse_array
+      spaces
+    rescue UnexpectedTokenError => e
+      raise e if debug
+    end
+    result
+  end
+  class UnexpectedTokenError < StandardError ; end
+
+  def initialize(options = {})
+    @debug = options.fetch(:debug, false)
+  end
+
+  private
+  attr_reader :input, :result, :debug
+
+  #### Non-Terminal Symbols
+
+  def parse_object
+    spaces
+    if next_is "{"
+      spaces
+      begin
+        parse_members
+      ensure # and recursively auto close
+        must_see "}"
+      end
+      spaces
+    end
+  end
+
+  def parse_members
+    parse_pair
+    spaces
+    if next_is ","
+      parse_members
+    end
+  end
+  
+  # An object's pair is the strictest of all
+  # elements: It has to have all syntax elements
+  # that's why we `&&`` them.
+  def parse_pair
+    parse_string &&
+    spaces &&
+    must_see(":") &&
+    spaces &&
+    # We may have to fill in a rhs *null*.
+    (parse_value || append('null'))
+  end
+
+  def parse_array
+    spaces
+    if next_is "["
+      spaces
+      begin
+        parse_elements || true
+       ensure # and recursively auto close
+         must_see "]"
+      end
+    end
+  end
+
+  def parse_elements
+    spaces
+    value_found = parse_value
+    spaces
+    while next_is ","
+      spaces
+      parse_value || append('null')
+      spaces
+    end
+
+    value_found
+  end
+
+  def parse_value
+    spaces
+    parse_string || parse_number || parse_object || parse_array || consume(TRUE) || consume(FALSE) || consume(NULL)
+  end
+  NULL  = /null/
+  FALSE = /false/
+  TRUE  = /true/
+ 
+  #### Terminal Symbols
+
+  def parse_string
+    spaces
+    if next_is "\""
+      consume CHAR_SEQUENCE
+
+      # We don't need to `ensure` here because we
+      # don't need to close strings recursively;
+      # a string cannot contain another string.
+
+      must_see "\""
+    end
+  end
+  CHAR_SEQUENCE = /([^\"\\\n]|\\([tbrfn\"\/\\])|\\u[0-9a-f]{4,4})*/i
+
+  def parse_number
+    spaces
+    number_found = parse_int
+    parse_frac
+    parse_exp
+
+    number_found
+  end
+
+  def parse_int
+    consume MINUS
+    consume DIGITS
+  end
+
+  def parse_frac
+    if next_is "."
+      consume DIGITS
+    end
+  end
+
+  def parse_exp
+    if next_is("e") || next_is("E")
+      consume(MINUS) || consume(PLUS)
+      consume DIGITS
+    end
+  end
+  MINUS = /-/
+  PLUS = /[+]/
+  DIGITS = /\d+/
+ 
+  #### Helpers 
+
+  def spaces
+    consume ANY_WHITESPACE
+  end
+  ANY_WHITESPACE = /\s*/
+
+  # `must_see` makes sure we add the necessary
+  # amount of (closing) tags to the output
+  def must_see(char)
+    spaces
+    if next_is char
+      # All good, input as expected.
+      char
+    elsif input.eos?
+      # Finally, the case we've been waiting for:
+      # Input exhausted, but still elements to close.
+      append char
+    else
+      append char
+      # The magic trick:
+      #   We implicitly use the stack to count how many closing tags we have/need.
+      #   We pair with exception's `ensure` mechanism to actually write them out.
+      if debug
+        msg = "Seen so far: %s" % result.clone
+        raise UnexpectedTokenError, msg
+      else
+        raise UnexpectedTokenError
+      end
+    end
+  end
+ 
+  # Checks whether the next char is `exptected_char`
+  #
+  # If it is, `expected_char` is consumed, `appended` and true is returned.
+  # If not, this call is a no op and false is returned.
+  def next_is(expected_char)
+    next_char = input.peek(1)
+    if next_char == expected_char
+      append input.getch
+    end
+  end
+
+  # `consume` is `next_is`'s sibling for arbitrary patterns
+  # not single chars. Same behavior.
+  def consume(pattern)
+    if input.match?(pattern)
+      append input.scan(pattern)
+    end
+  end
+
+  def append(str)
+    result << str
+  end
+end
+

data/lib/drjson/version.rb

@@ -0,0 +1,3 @@
+module Drjson
+  VERSION = "1.0.0"
+end

data/spec/drjson_spec.rb

@@ -0,0 +1,178 @@
+
+require 'rspec'
+require 'json'
+require 'yajl'
+require 'oj'
+
+require 'drjson'
+
+describe DrJson do
+  
+  def doctor
+    DrJson.new(:debug => true)
+  end
+
+  describe "completion" do
+  it "works" do
+    doctor.repair("{").should == "{}"
+  end
+
+  it "insert the object value if absent" do
+    doctor.repair('{"foo": ').should == '{"foo": null}'
+  end
+
+  it "closes array brackets" do
+    doctor.repair("[42").should == "[42]"
+  end
+  it "inserts missing trailing array elements" do
+    doctor.repair("[42,").should == "[42,null]"
+    doctor.repair("[42,7,").should == "[42,7,null]"
+  end
+
+  it "works" do
+    doctor.repair('{"foo": [42 ').should == '{"foo": [42 ]}'
+    doctor.repair('{"foo": [ ').should == '{"foo": [ ]}'
+  end
+  it "works" do
+    doctor.repair('{"foo": "bar').should == '{"foo": "bar"}'
+    doctor.repair('{"foo": "ba').should == '{"foo": "ba"}'
+
+    doctor.repair('{"foo": "').should == '{"foo": ""}'
+    doctor.repair('{"foo": ').should == '{"foo": null}'
+    doctor.repair('{"foo" ').should == '{"foo" :null}'
+    doctor.repair('{"foo ').should == '{"foo ":null}'
+    doctor.repair('{"foo').should == '{"foo":null}'
+
+    doctor.repair('{"f').should == '{"f":null}'
+    doctor.repair('{"').should == '{"":null}'
+    doctor.repair('{').should == '{}'
+  end
+   end #completion
+
+  ##################
+  it "works" do
+    doctor.repair("[]").should == "[]"
+    doctor.repair("[42]").should == "[42]"
+    doctor.repair('{"foo": "bar"}').should == '{"foo": "bar"}'
+  end
+  it "empty strings" do
+    doctor.repair('{"":""}').should == '{"":""}'
+  end
+ it "works" do
+    doctor.repair('{"foo": 42 }').should == '{"foo": 42 }'
+    doctor.repair('{"foo": 42}').should == '{"foo": 42}'
+  end
+  it "multiline completion" do
+    doctor.repair('{"foo": "bar').should == '{"foo": "bar"}'
+  end
+  it "works" do
+    doctor.repair('{"foo": 42, "bar": 7 }').should == '{"foo": 42, "bar": 7 }'
+  end
+  it "works" do
+    doctor.repair('{"foo": {"bar" : "baz"} }').should == '{"foo": {"bar" : "baz"} }'
+    doctor.repair('{"foo": {"bar" : "baz').should == '{"foo": {"bar" : "baz"}}'
+  end
+  it "works" do
+    doctor.repair('{"foo": [] }').should == '{"foo": [] }'
+    doctor.repair('{"foo": [42] }').should == '{"foo": [42] }'
+  end
+  it "handles inclomplete (null|false|true) tokens" do
+    DrJson.new.repair('[tru').should == '[]'
+    DrJson.new.repair('[fals').should == '[]'
+    DrJson.new.repair('[nul').should == '[]'
+  end
+  it "handles inclomplete (null|false|true) tokens" do
+    DrJson.new.repair('[[tru').should == '[[]]'
+    DrJson.new.repair('[[[fal').should == '[[[]]]'
+    DrJson.new.repair('[[[[nul').should == '[[[[]]]]'
+  end
+   it "handles inclomplete (null|false|true) values" do
+    DrJson.new.repair('{"foo":tru').should == '{"foo":null}'
+    DrJson.new.repair('{"foo":{"bar":tru').should == '{"foo":{"bar":null}}'
+  end
+    it "handles inclomplete keys in pairs" do
+    DrJson.new.repair('{"foo').should == '{"foo":null}'
+  end
+  it "works" do
+    doctor.repair('{"foo": [42,7] }').should == '{"foo": [42,7] }'
+    doctor.repair('{"foo": [42, 7] }').should == '{"foo": [42, 7] }'
+    doctor.repair('{"foo": [42 ,7] }').should == '{"foo": [42 ,7] }'
+    doctor.repair('{"foo": [42 , 7] }').should == '{"foo": [42 , 7] }'
+    doctor.repair('{"foo": [42 , 7, 4711] }').should == '{"foo": [42 , 7, 4711] }'
+  end
+
+  it "supports all the null|false|true terminal symbols" do
+    doctor.repair('[null]').should == '[null]'
+    doctor.repair('[true]').should == '[true]'
+    doctor.repair('[false]').should == '[false]'
+  end
+
+  context "numbers" do
+    it "supports -" do
+      doctor.repair('{"foo": -42 }').should == '{"foo": -42 }'
+    end
+    it "supports floats" do
+      doctor.repair('{"foo": 7.42 }').should == '{"foo": 7.42 }'
+    end
+    it "supports exponents" do
+      doctor.repair('{"foo": 1e-4 }').should == '{"foo": 1e-4 }'
+      doctor.repair('{"foo": 1E-4 }').should == '{"foo": 1E-4 }'
+      doctor.repair('{"foo": 1e+4 }').should == '{"foo": 1e+4 }'
+      doctor.repair('{"foo": 1E+4 }').should == '{"foo": 1E+4 }'
+      doctor.repair('{"foo": 1e4 }').should == '{"foo": 1e4 }'
+      doctor.repair('{"foo": 1E4 }').should == '{"foo": 1E4 }'
+    end
+  end
+
+    it "knows escape sequences" do
+      doctor.repair('{"foo": "\"" }').should == '{"foo": "\"" }'
+      doctor.repair('{"foo": "\\\\" }').should == '{"foo": "\\\\" }'
+      doctor.repair('{"foo": "\/" }').should == '{"foo": "\/" }'
+      doctor.repair('{"foo": "\b" }').should == '{"foo": "\b" }'
+      doctor.repair('{"foo": "\f" }').should == '{"foo": "\f" }'
+      doctor.repair('{"foo": "\n" }').should == '{"foo": "\n" }'
+      doctor.repair('{"foo": "\r" }').should == '{"foo": "\r" }'
+      doctor.repair('{"foo": "\t" }').should == '{"foo": "\t" }'
+      doctor.repair('{"foo": "\u4711" }').should == '{"foo": "\u4711" }'
+      doctor.repair('{"foo": "\ubeef" }').should == '{"foo": "\ubeef" }'
+    end
+
+  it "does not break on unexpected input, it tries to fix it" do
+    broken_json ='{"foo": ["beef" {][] senseless'
+    repaired_json = '{"foo": ["beef" ]}'
+    DrJson.new.repair(broken_json).should == repaired_json
+  end
+
+  it "indicates unexpected input in debug mode" do
+    broken_json ='{"foo": "beef" {'
+    lambda {DrJson.new(:debug => true).repair(broken_json)}.should raise_error DrJson::UnexpectedTokenError
+  end
+
+  context "pass-through behavior for real life files" do
+    json_files = Dir.glob("spec/fixtures/yajl-ruby/*.json")
+
+    json_files.each do |file|
+    describe "#{file}" do
+      #it "is parseable with JSON: #{file}" do
+      #  JSON.parse(File.read(file))
+      #end
+      it "is parseable with Oj: #{file}" do
+        Oj.load(File.read(file))
+      end
+      #it "is parseable with Yajl: #{file}" do
+      #  Yajl::Parser.parse(File.read(file))
+      #end
+      it "does pass through correct, real life file #{file}" do
+        test_file(file)
+      end
+    end
+    end
+  end
+
+  def test_file(file_name)
+    json_str = File.read (file_name)
+    repaired = doctor.repair(json_str)
+    repaired.should == json_str
+  end
+end
+

metadata

@@ -0,0 +1,57 @@
+--- !ruby/object:Gem::Specification
+name: drjson
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Matthias Luedtke
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-09-23 00:00:00.000000000 Z
+dependencies: []
+description: Closes abruptly cut-off JSON strings.
+email:
+- github@matthias.luedtke.de
+executables:
+- drjson
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/drjson
+- drjson.gemspec
+- lib/drjson.rb
+- lib/drjson/version.rb
+- spec/drjson_spec.rb
+homepage: http://drjson.github.com/
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Closes abruptly cut-off JSON strings.
+test_files:
+- spec/drjson_spec.rb