json 0.4.3 → 1.0.0

data/CHANGES

@@ -1,3 +1,8 @@
+2007-03-24 (1.0.0)
+  * Added C implementations for the JSON generator and a ragel based JSON
+    parser in C.
+  * Much more tests, especially fixtures from json.org.
+  * Further improved conformance to RFC4627.
 2007-02-09 (0.4.3)
   * Conform more to RFC4627 for JSON: This means JSON strings
     now always must contain exactly one object "{ ... }" or array "[ ... ]" in
@@ -14,7 +19,7 @@
   * Fixed a bug in handling solidi (/-characters), that was reported by
     Kevin Gilpin <kevin.gilpin@alum.mit.edu>.
 2006-02-06 (0.4.1)
-  * Fixed a bug concerning escaping with backslashes. Thanks for the report go
+  * Fixed a bug related to escaping with backslashes. Thanks for the report go
     to Florian Munz <surf@theflow.de>.
 2005-09-23 (0.4.0)
   * Initial Rubyforge Version

data/README

@@ -1,27 +1,69 @@
+Dependencies for Building
+=========================
+
+- You need rake to build the extensions and install them.
+
+  You can get it from rubyforge:
+    http://rubyforge.org/projects/rake
+
+  or just type
+
+  # gem install rake
+
+  for the installation via rubygems.
+
+- If you want to rebuild the parser.c file or draw nice graphviz images of the
+  state machines, you need ragel from:
+    http://www.cs.queensu.ca/~thurston/ragel
+
 Installation
 ============
 
+It's recommended to use the extension variant of JSON, because it's quite a bit
+faster than the pure ruby variant. If you cannot build it on your system, you
+can settle for the latter.
+
 Just type into the command line as root:
 
-# ruby install.rb
+# rake install
+
+The above command will build the extensions and install them on your system.
+
+# rake install_pure
 
 or
 
-# rake install
+# ruby install.rb
+
+will just install the pure ruby implementation of JSON.
+
+If you use Rubygems you can type
+
+# gem install json
+
+instead, to install the newest JSON version.
+
+There is also a pure ruby json only variant of the gem, that can be installed
+with:
+
+# gem install json_pure
 
 Testing and Examples
 ====================
 
 To run the tests type:
 
-$ ruby -I lib tests/runner.rb
+$ rake test_ext
 
-or
+This will build the extensions first and then test them.
+
+$ rake test_pure
 
-$ rake test
+This will test the pure ruby extensions.
 
-To get an idea how this library is used also look at the tests (until I have
-time to better document it.)
+There is also a small example in tools/server.rb if you want to see, how
+receiving a JSON object from a webrick server in your browser with the
+javasript prototype library (http://www.prototypejs.org) works.
 
 Author
 ======

data/Rakefile

@@ -1,86 +1,250 @@
+# vim: set et sw=2 ts=2:
 require 'rake/gempackagetask'
+require 'rake/clean'
 
 require 'rbconfig'
 include Config
 
-PKG_NAME = 'json'
-PKG_VERSION = File.read('VERSION').chomp
-PKG_FILES = FileList["**/*"].exclude(/CVS|pkg|coverage/)
+PKG_NAME          = 'json'
+PKG_VERSION       = File.read('VERSION').chomp
+PKG_FILES         = FileList["**/*"].exclude(/CVS|pkg|coverage/)
+EXT_ROOT_DIR      = 'ext/json/ext'
+EXT_PARSER_DIR    = "#{EXT_ROOT_DIR}/parser"
+EXT_PARSER_DL     = "#{EXT_ROOT_DIR}/parser.#{CONFIG['DLEXT']}"
+EXT_PARSER_SRC    = "#{EXT_PARSER_DIR}/parser.c"
+PKG_FILES << EXT_PARSER_SRC
+EXT_GENERATOR_DIR = "#{EXT_ROOT_DIR}/generator"
+EXT_GENERATOR_DL  = "#{EXT_ROOT_DIR}/generator.#{CONFIG['DLEXT']}"
+EXT_GENERATOR_SRC = "#{EXT_GENERATOR_DIR}/generator.c"
+RAGEL_CODEGEN     = %w[rlcodegen rlgen-cd].find { |c| system(c, '-v') }
+RAGEL_PATH        = "#{EXT_PARSER_DIR}/parser.rl"
+CLEAN.include 'doc', 'coverage', FileList['diagrams/*.*'],
+  FileList["ext/**/*.{so,bundle,#{CONFIG['DLEXT']},o}"]
+
+desc "Installing library (pure)"
+task :install_pure => :version do
+  ruby 'install.rb'
+end
 
-desc "Installing library"
-task :install  do
-    ruby 'install.rb'
+desc "Installing library (extension)"
+task :install_ext => [ :compile, :install_pure ]  do
+  sitearchdir = CONFIG["sitearchdir"]
+  cd 'ext' do
+    for file in Dir["json/ext/*.#{CONFIG['DLEXT']}"]
+      d = File.join(sitearchdir, file)
+      mkdir_p File.dirname(d)
+      install(file, d)
+    end
+  end
 end
 
-desc "Testing library"
-task :test do
-    ruby 'tests/runner.rb'
+task :install => :install_ext
+
+desc "Compiling extension"
+task :compile => [ EXT_PARSER_DL, EXT_GENERATOR_DL ]
+
+file EXT_PARSER_DL => EXT_PARSER_SRC do
+  cd EXT_PARSER_DIR do
+    ruby 'extconf.rb'
+    sh 'make'
+  end
+  cp "#{EXT_PARSER_DIR}/parser.#{CONFIG['DLEXT']}", EXT_ROOT_DIR
 end
 
-desc "Testing library with coverage"
-task :coverage do
-    system 'rcov -x tests -Ilib tests/runner.rb'
+file EXT_GENERATOR_DL => EXT_GENERATOR_SRC do
+  cd EXT_GENERATOR_DIR do
+    ruby 'extconf.rb'
+    sh 'make'
+  end
+  cp "#{EXT_GENERATOR_DIR}/generator.#{CONFIG['DLEXT']}", EXT_ROOT_DIR
+end
+
+desc "Generate parser with ragel"
+task :ragel => EXT_PARSER_SRC
+
+task :ragel_clean do
+  rm_rf EXT_PARSER_SRC
+end
+
+file EXT_PARSER_SRC => RAGEL_PATH do
+  cd EXT_PARSER_DIR do
+    sh "ragel parser.rl | #{RAGEL_CODEGEN} -G2"
+  end
+end
+
+desc "Generate diagrams of ragel parser (ps)"
+task :ragel_dot_ps do
+  root = 'diagrams'
+  cd EXT_PARSER_DIR do
+    specs = []
+    File.new('parser.rl').grep(/^\s*machine\s*(\S+);\s*$/) { specs << $1 }
+    for s in specs 
+      sh "ragel parser.rl -S#{s} | #{RAGEL_CODEGEN} -p -V|dot -Tps -o#{root}/#{s}.ps"
+    end
+  end
+end
+
+desc "Generate diagrams of ragel parser (png)"
+task :ragel_dot_png do
+  root = 'diagrams'
+  cd EXT_PARSER_DIR do
+    specs = []
+    File.new('parser.rl').grep(/^\s*machine\s*(\S+);\s*$/) { specs << $1 }
+    for s in specs 
+      sh "ragel parser.rl -S#{s} | #{RAGEL_CODEGEN} -p -V|dot -Tpng -o#{root}/#{s}.png"
+    end
+  end
+end
+
+desc "Generate diagrams of ragel parser"
+task :ragel_dot => [ :ragel_dot_png, :ragel_dot_ps ]
+
+desc "Testing library (pure ruby)"
+task :test_pure => :clean do
+  ruby '-I lib tests/runner.rb'
 end
-desc "Removing generated files"
-task :clean do
-  rm_rf 'doc'
+
+desc "Testing library (extension)"
+task :test_ext => :compile do
+  ruby '-I ext:lib tests/runner.rb'
 end
 
-# Create RDOC documentation.
-task :doc do
-    sh 'rdoc -d -S -o doc lib/json.rb lib/json/editor.rb'
+desc "Benchmarking parser (pure)"
+task :benchmark_parser_pure do
+  ruby '-I lib benchmarks/benchmark_parser.rb'
 end
 
-spec = Gem::Specification.new do |s|
-    #### Basic information.
+desc "Benchmarking generator (pure)"
+task :benchmark_generator_pure do
+  ruby '-I lib benchmarks/benchmark_generator.rb'
+  ruby 'benchmarks/benchmark_rails.rb'
+end
 
-    s.name = 'json'
-    s.version = PKG_VERSION
-    s.summary = "A JSON implementation in Ruby"
-    s.description = ""
+desc "Benchmarking library (pure)"
+task :benchmark_pure => [ :benchmark_parser_pure, :benchmark_generator_pure ]
 
-    #### Dependencies and requirements.
+desc "Benchmarking parser (extension)"
+task :benchmark_parser_ext => :compile do
+  ruby '-I ext:lib benchmarks/benchmark_parser.rb'
+end
 
-    #s.add_dependency('log4r', '> 1.0.4')
-    #s.requirements << ""
+desc "Benchmarking generator (extension)"
+task :benchmark_generator_ext => :compile do
+  ruby '-I ext:lib benchmarks/benchmark_generator.rb'
+  ruby 'benchmarks/benchmark_rails.rb'
+end
 
-    s.files = PKG_FILES
+desc "Benchmarking library (extension)"
+task :benchmark_ext => [ :benchmark_parser_ext, :benchmark_generator_ext ]
 
-    #### C code extensions.
+task :benchmark do
+  puts "Benchmarking extension variant"
+  Rake::Task[:benchmark_ext].invoke
+  puts "Benchmarking pure variant"
+  Rake::Task[:benchmark_pure].invoke
+end
 
-    #s.extensions << "ext/extconf.rb"
+desc "Testing library with coverage" # XXX broken
+task :coverage do
+  system 'RUBYOPT="" rcov -x tests -Ilib tests/runner.rb'
+end
 
-    #### Load-time details: library and application (you will need one or both).
+desc "Create RDOC documentation"
+task :doc => [ :version, EXT_PARSER_SRC ] do
+  sh "rdoc -m JSON -d -S -o doc #{FileList['lib/**/*.rb']} #{EXT_PARSER_SRC} #{EXT_GENERATOR_SRC}"
+end
 
-    s.require_path = 'lib'                         # Use these for libraries.
-    #s.autorequire = 'json'
+spec_pure = Gem::Specification.new do |s|
+  s.name = 'json_pure'
+  s.version = PKG_VERSION
+  s.summary = "A JSON implementation in Ruby"
+  s.description = ""
 
-    s.bindir = "bin"                               # Use these for applications.
-    s.executables = ["edit_json.rb"]
-    s.default_executable = "edit_json.rb"
+  s.files = PKG_FILES
 
-    #### Documentation and testing.
+  s.require_path = 'lib'
 
-    s.has_rdoc = true
-    #s.extra_rdoc_files = rd.rdoc_files.reject { |fn| fn =~ /\.rb$/ }.to_a
-    #s.rdoc_options <<
-    #  '--title' <<  'Rake -- Ruby Make' <<
-    #  '--main' << 'README' <<
-    #  '--line-numbers'
-    s.test_files << 'tests/runner.rb'
+  s.bindir = "bin"
+  s.executables = ["edit_json.rb"]
+  s.default_executable = "edit_json.rb"
 
-    #### Author and project details.
+  s.has_rdoc = true
+  s.rdoc_options <<
+    '--title' <<  'JSON -- A JSON implemention' <<
+    '--main' << 'JSON' << '--line-numbers'
+  s.test_files << 'tests/runner.rb'
 
-    s.author = "Florian Frank"
-    s.email = "flori@ping.de"
-    s.homepage = "http://json.rubyforge.org"
-    s.rubyforge_project = "json"
+  s.author = "Florian Frank"
+  s.email = "flori@ping.de"
+  s.homepage = "http://json.rubyforge.org"
+  s.rubyforge_project = "json"
 end
 
-Rake::GemPackageTask.new(spec) do |pkg|
+Rake::GemPackageTask.new(spec_pure) do |pkg|
     pkg.need_tar = true
     pkg.package_files += PKG_FILES
 end
 
-task :release => [ :clean, :package ]
-    # vim: set et sw=4 ts=4:
+spec_ext = Gem::Specification.new do |s|
+  s.name = 'json'
+  s.version = PKG_VERSION
+  s.summary = "A JSON implementation as a Ruby extension"
+  s.description = ""
+
+  s.files = PKG_FILES
+
+  s.extensions <<
+    "#{EXT_PARSER_DIR}/extconf.rb" <<
+    "#{EXT_GENERATOR_DIR}/extconf.rb"
+
+  s.require_path = EXT_ROOT_DIR
+  s.require_paths << 'ext'
+  s.require_paths << 'lib'
+
+  s.bindir = "bin"
+  s.executables = ["edit_json.rb"]
+  s.default_executable = "edit_json.rb"
+
+  s.has_rdoc = true
+  s.rdoc_options <<
+    '--title' <<  'JSON -- A JSON implemention' <<
+    '--main' << 'JSON' << '--line-numbers'
+  s.test_files << 'tests/runner.rb'
+
+  s.author = "Florian Frank"
+  s.email = "flori@ping.de"
+  s.homepage = "http://json.rubyforge.org"
+  s.rubyforge_project = "json"
+end
+
+Rake::GemPackageTask.new(spec_ext) do |pkg|
+  pkg.need_tar = true
+  pkg.package_files += PKG_FILES
+end
+
+task :mrproper => [ :ragel_clean, :clean ] do
+  for dir in [ EXT_PARSER_DIR, EXT_GENERATOR_DIR ]
+    cd(dir) {  rm_f 'Makefile' }
+  end
+end
+
+desc m = "Writing version information for #{PKG_VERSION}"
+task :version do
+  puts m
+  File.open(File.join('lib', 'json', 'version.rb'), 'w') do |v|
+    v.puts <<EOT
+module JSON
+  # JSON version
+  VERSION         = '#{PKG_VERSION}'
+  VERSION_ARRAY   = VERSION.split(/\\./).map { |x| x.to_i } # :nodoc:
+  VERSION_MAJOR   = VERSION_ARRAY[0] # :nodoc:
+  VERSION_MINOR   = VERSION_ARRAY[1] # :nodoc:
+  VERSION_BUILD   = VERSION_ARRAY[2] # :nodoc:
+end
+EOT
+  end
+end
+
+task :release => [ :version, :clean, :ragel_clean, :ragel, :package ]
+
+task :default => [ :version, :compile ]

data/TODO

@@ -0,0 +1 @@
+* Get the extension to compile on Windows, eventually?

data/VERSION

@@ -1 +1 @@
-0.4.3
+1.0.0

data/benchmarks/benchmark.txt

@@ -0,0 +1,133 @@
+(in /home/flori/json)
+Ragel Code Generator version 5.17 January 2007
+Copyright (c) 2001-2006 by Adrian Thurston
+Benchmarking extension variant
+First run for warmup.
+Running 'BC_Parser' for a duration of 10 secs per method:
+              real      total      utime      stime     cutime     cstime
+parser:   10.005944  10.010000   9.970000   0.040000   0.000000   0.000000
+              5091  508.591409    0.001966
+             calls   calls/sec   secs/call
+--------------------------------------------------------------------------------
+Running 'BC_Parser' for a duration of 10 secs per method:
+              real      total      utime      stime     cutime     cstime
+parser:   10.006881  10.000000   9.950000   0.050000   0.000000   0.000000
+              5055  505.500000    0.001978
+             calls   calls/sec   secs/call
+--------------------------------------------------------------------------------
+================================================================================
+
+
+[null,false,true,"f\u00d6\u00df\u00c4r",["n\u20acst\u20acd",true],{"quux":true,"foo\u00df":"b\u00e4r"}]
+First run for warmup.
+Running 'BC_Generator' for a duration of 10 secs per method:
+                        real      total      utime      stime     cutime     cstime
+generator_pretty:   10.001066  10.010000   9.990000   0.020000   0.000000   0.000000
+                        3492  348.851149    0.002867
+generator_fast  :   10.002910  10.000000   9.980000   0.020000   0.000000   0.000000
+                        5416  541.600000    0.001846
+generator_safe  :   10.003107  10.010000  10.000000   0.010000   0.000000   0.000000
+                        4926  492.107892    0.002032
+                       calls   calls/sec   secs/call
+--------------------------------------------------------------------------------
+Running 'BC_Generator' for a duration of 10 secs per method:
+                        real      total      utime      stime     cutime     cstime
+generator_pretty:   10.009150  10.010000  10.010000   0.000000   0.000000   0.000000
+                        3511  350.749251    0.002851
+generator_fast  :   10.014407  10.020000  10.020000   0.000000   0.000000   0.000000
+                        5411  540.019960    0.001852
+generator_safe  :   10.010055  10.000000  10.000000   0.000000   0.000000   0.000000
+                        4933  493.300000    0.002027
+                       calls   calls/sec   secs/call
+--------------------------------------------------------------------------------
+================================================================================
+
+Comparison in BC_Generator:
+                  secs/call     speed
+generator_fast  :    0.002 ->    1.540x
+generator_safe  :    0.002 ->    1.406x
+generator_pretty:    0.003 ->    1.000x
+--------------------------------------------------------------------------------
+
+[null, false, true, "f\u00d6\u00df\u00c4r", ["n\u20acst\u20acd", true], {quux: true, "foo\u00df": "b\u00e4r"}]
+First run for warmup.
+Running 'BC_Rails' for a duration of 10 secs per method:
+                 real      total      utime      stime     cutime     cstime
+generator:   10.026043  10.020000  10.020000   0.000000   0.000000   0.000000
+                  239   23.852295    0.041925
+                calls   calls/sec   secs/call
+--------------------------------------------------------------------------------
+Running 'BC_Rails' for a duration of 10 secs per method:
+                 real      total      utime      stime     cutime     cstime
+generator:   10.010931  10.020000  10.010000   0.010000   0.000000   0.000000
+                  238   23.752495    0.042101
+                calls   calls/sec   secs/call
+--------------------------------------------------------------------------------
+================================================================================
+
+
+Benchmarking pure variant
+First run for warmup.
+Running 'BC_Parser' for a duration of 10 secs per method:
+              real      total      utime      stime     cutime     cstime
+parser:   10.022352  10.030000  10.020000   0.010000   0.000000   0.000000
+               288   28.713858    0.034826
+             calls   calls/sec   secs/call
+--------------------------------------------------------------------------------
+Running 'BC_Parser' for a duration of 10 secs per method:
+              real      total      utime      stime     cutime     cstime
+parser:   10.006552  10.000000  10.000000   0.000000   0.000000   0.000000
+               289   28.900000    0.034602
+             calls   calls/sec   secs/call
+--------------------------------------------------------------------------------
+================================================================================
+
+
+[null,false,true,"f\u00d6\u00df\u00c4r",["n\u20acst\u20acd",true],{"quux":true,"foo\u00df":"b\u00e4r"}]
+First run for warmup.
+Running 'BC_Generator' for a duration of 10 secs per method:
+                        real      total      utime      stime     cutime     cstime
+generator_fast  :   10.011644  10.010000  10.010000   0.000000   0.000000   0.000000
+                         411   41.058941    0.024355
+generator_safe  :   10.007100  10.010000  10.010000   0.000000   0.000000   0.000000
+                         352   35.164835    0.028438
+generator_pretty:   10.008156  10.010000  10.010000   0.000000   0.000000   0.000000
+                         341   34.065934    0.029355
+                       calls   calls/sec   secs/call
+--------------------------------------------------------------------------------
+Running 'BC_Generator' for a duration of 10 secs per method:
+                        real      total      utime      stime     cutime     cstime
+generator_fast  :   10.005185  10.010000  10.010000   0.000000   0.000000   0.000000
+                         411   41.058941    0.024355
+generator_safe  :   10.006932  10.010000  10.010000   0.000000   0.000000   0.000000
+                         351   35.064935    0.028519
+generator_pretty:   10.007414  10.000000  10.000000   0.000000   0.000000   0.000000
+                         340   34.000000    0.029412
+                       calls   calls/sec   secs/call
+--------------------------------------------------------------------------------
+================================================================================
+
+Comparison in BC_Generator:
+                  secs/call     speed
+generator_fast  :    0.024 ->    1.208x
+generator_safe  :    0.029 ->    1.031x
+generator_pretty:    0.029 ->    1.000x
+--------------------------------------------------------------------------------
+
+[null, false, true, "f\u00d6\u00df\u00c4r", ["n\u20acst\u20acd", true], {quux: true, "foo\u00df": "b\u00e4r"}]
+First run for warmup.
+Running 'BC_Rails' for a duration of 10 secs per method:
+                 real      total      utime      stime     cutime     cstime
+generator:   10.005748  10.000000  10.000000   0.000000   0.000000   0.000000
+                  240   24.000000    0.041667
+                calls   calls/sec   secs/call
+--------------------------------------------------------------------------------
+Running 'BC_Rails' for a duration of 10 secs per method:
+                 real      total      utime      stime     cutime     cstime
+generator:   10.006764  10.010000  10.010000   0.000000   0.000000   0.000000
+                  239   23.876124    0.041883
+                calls   calls/sec   secs/call
+--------------------------------------------------------------------------------
+================================================================================
+
+