filipegiusti-yajl-ruby 0.6.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (138) hide show
  1. data/.gitignore +7 -0
  2. data/CHANGELOG.md +217 -0
  3. data/MIT-LICENSE +20 -0
  4. data/README.rdoc +315 -0
  5. data/Rakefile +35 -0
  6. data/VERSION.yml +4 -0
  7. data/benchmark/encode.rb +46 -0
  8. data/benchmark/encode_json_and_marshal.rb +35 -0
  9. data/benchmark/encode_json_and_yaml.rb +47 -0
  10. data/benchmark/http.rb +30 -0
  11. data/benchmark/parse.rb +49 -0
  12. data/benchmark/parse_json_and_marshal.rb +47 -0
  13. data/benchmark/parse_json_and_yaml.rb +56 -0
  14. data/benchmark/parse_stream.rb +48 -0
  15. data/benchmark/subjects/item.json +1 -0
  16. data/benchmark/subjects/ohai.json +1216 -0
  17. data/benchmark/subjects/ohai.marshal_dump +0 -0
  18. data/benchmark/subjects/ohai.yml +975 -0
  19. data/benchmark/subjects/twitter_search.json +1 -0
  20. data/benchmark/subjects/twitter_stream.json +430 -0
  21. data/benchmark/subjects/unicode.json +1 -0
  22. data/examples/encoding/chunked_encoding.rb +27 -0
  23. data/examples/encoding/one_shot.rb +13 -0
  24. data/examples/encoding/to_an_io.rb +12 -0
  25. data/examples/http/twitter_search_api.rb +15 -0
  26. data/examples/http/twitter_stream_api.rb +27 -0
  27. data/examples/parsing/from_file.rb +14 -0
  28. data/examples/parsing/from_stdin.rb +9 -0
  29. data/examples/parsing/from_string.rb +15 -0
  30. data/ext/api/yajl_common.h +85 -0
  31. data/ext/api/yajl_gen.h +130 -0
  32. data/ext/api/yajl_parse.h +182 -0
  33. data/ext/extconf.rb +8 -0
  34. data/ext/yajl.c +157 -0
  35. data/ext/yajl_alloc.c +65 -0
  36. data/ext/yajl_alloc.h +50 -0
  37. data/ext/yajl_buf.c +119 -0
  38. data/ext/yajl_buf.h +73 -0
  39. data/ext/yajl_bytestack.h +85 -0
  40. data/ext/yajl_encode.c +179 -0
  41. data/ext/yajl_encode.h +44 -0
  42. data/ext/yajl_ext.c +830 -0
  43. data/ext/yajl_ext.h +97 -0
  44. data/ext/yajl_gen.c +298 -0
  45. data/ext/yajl_lex.c +744 -0
  46. data/ext/yajl_lex.h +135 -0
  47. data/ext/yajl_parser.c +447 -0
  48. data/ext/yajl_parser.h +79 -0
  49. data/lib/yajl.rb +82 -0
  50. data/lib/yajl/bzip2.rb +11 -0
  51. data/lib/yajl/bzip2/stream_reader.rb +29 -0
  52. data/lib/yajl/bzip2/stream_writer.rb +15 -0
  53. data/lib/yajl/deflate.rb +6 -0
  54. data/lib/yajl/deflate/stream_reader.rb +38 -0
  55. data/lib/yajl/deflate/stream_writer.rb +21 -0
  56. data/lib/yajl/gzip.rb +6 -0
  57. data/lib/yajl/gzip/stream_reader.rb +28 -0
  58. data/lib/yajl/gzip/stream_writer.rb +14 -0
  59. data/lib/yajl/http_stream.rb +140 -0
  60. data/lib/yajl/json_gem.rb +14 -0
  61. data/lib/yajl/json_gem/encoding.rb +50 -0
  62. data/lib/yajl/json_gem/parsing.rb +27 -0
  63. data/spec/encoding/encoding_spec.rb +212 -0
  64. data/spec/http/fixtures/http.bzip2.dump +0 -0
  65. data/spec/http/fixtures/http.deflate.dump +0 -0
  66. data/spec/http/fixtures/http.gzip.dump +0 -0
  67. data/spec/http/fixtures/http.raw.dump +1226 -0
  68. data/spec/http/http_delete_spec.rb +91 -0
  69. data/spec/http/http_get_spec.rb +91 -0
  70. data/spec/http/http_post_spec.rb +110 -0
  71. data/spec/http/http_put_spec.rb +98 -0
  72. data/spec/json_gem_compatibility/compatibility_spec.rb +204 -0
  73. data/spec/parsing/active_support_spec.rb +64 -0
  74. data/spec/parsing/chunked_spec.rb +98 -0
  75. data/spec/parsing/fixtures/fail.15.json +1 -0
  76. data/spec/parsing/fixtures/fail.16.json +1 -0
  77. data/spec/parsing/fixtures/fail.17.json +1 -0
  78. data/spec/parsing/fixtures/fail.26.json +1 -0
  79. data/spec/parsing/fixtures/fail11.json +1 -0
  80. data/spec/parsing/fixtures/fail12.json +1 -0
  81. data/spec/parsing/fixtures/fail13.json +1 -0
  82. data/spec/parsing/fixtures/fail14.json +1 -0
  83. data/spec/parsing/fixtures/fail19.json +1 -0
  84. data/spec/parsing/fixtures/fail20.json +1 -0
  85. data/spec/parsing/fixtures/fail21.json +1 -0
  86. data/spec/parsing/fixtures/fail22.json +1 -0
  87. data/spec/parsing/fixtures/fail23.json +1 -0
  88. data/spec/parsing/fixtures/fail24.json +1 -0
  89. data/spec/parsing/fixtures/fail25.json +1 -0
  90. data/spec/parsing/fixtures/fail27.json +2 -0
  91. data/spec/parsing/fixtures/fail28.json +2 -0
  92. data/spec/parsing/fixtures/fail3.json +1 -0
  93. data/spec/parsing/fixtures/fail4.json +1 -0
  94. data/spec/parsing/fixtures/fail5.json +1 -0
  95. data/spec/parsing/fixtures/fail6.json +1 -0
  96. data/spec/parsing/fixtures/fail9.json +1 -0
  97. data/spec/parsing/fixtures/pass.array.json +6 -0
  98. data/spec/parsing/fixtures/pass.codepoints_from_unicode_org.json +1 -0
  99. data/spec/parsing/fixtures/pass.contacts.json +1 -0
  100. data/spec/parsing/fixtures/pass.db100.xml.json +1 -0
  101. data/spec/parsing/fixtures/pass.db1000.xml.json +1 -0
  102. data/spec/parsing/fixtures/pass.dc_simple_with_comments.json +11 -0
  103. data/spec/parsing/fixtures/pass.deep_arrays.json +1 -0
  104. data/spec/parsing/fixtures/pass.difficult_json_c_test_case.json +1 -0
  105. data/spec/parsing/fixtures/pass.difficult_json_c_test_case_with_comments.json +1 -0
  106. data/spec/parsing/fixtures/pass.doubles.json +1 -0
  107. data/spec/parsing/fixtures/pass.empty_array.json +1 -0
  108. data/spec/parsing/fixtures/pass.empty_string.json +1 -0
  109. data/spec/parsing/fixtures/pass.escaped_bulgarian.json +4 -0
  110. data/spec/parsing/fixtures/pass.escaped_foobar.json +1 -0
  111. data/spec/parsing/fixtures/pass.item.json +1 -0
  112. data/spec/parsing/fixtures/pass.json-org-sample1.json +23 -0
  113. data/spec/parsing/fixtures/pass.json-org-sample2.json +11 -0
  114. data/spec/parsing/fixtures/pass.json-org-sample3.json +26 -0
  115. data/spec/parsing/fixtures/pass.json-org-sample4-nows.json +88 -0
  116. data/spec/parsing/fixtures/pass.json-org-sample4.json +89 -0
  117. data/spec/parsing/fixtures/pass.json-org-sample5.json +27 -0
  118. data/spec/parsing/fixtures/pass.map-spain.xml.json +1 -0
  119. data/spec/parsing/fixtures/pass.ns-invoice100.xml.json +1 -0
  120. data/spec/parsing/fixtures/pass.ns-soap.xml.json +1 -0
  121. data/spec/parsing/fixtures/pass.numbers-fp-4k.json +6 -0
  122. data/spec/parsing/fixtures/pass.numbers-fp-64k.json +61 -0
  123. data/spec/parsing/fixtures/pass.numbers-int-4k.json +11 -0
  124. data/spec/parsing/fixtures/pass.numbers-int-64k.json +154 -0
  125. data/spec/parsing/fixtures/pass.twitter-search.json +1 -0
  126. data/spec/parsing/fixtures/pass.twitter-search2.json +1 -0
  127. data/spec/parsing/fixtures/pass.unicode.json +3315 -0
  128. data/spec/parsing/fixtures/pass.yelp.json +1 -0
  129. data/spec/parsing/fixtures/pass1.json +56 -0
  130. data/spec/parsing/fixtures/pass2.json +1 -0
  131. data/spec/parsing/fixtures/pass3.json +6 -0
  132. data/spec/parsing/fixtures_spec.rb +41 -0
  133. data/spec/parsing/one_off_spec.rb +54 -0
  134. data/spec/rcov.opts +3 -0
  135. data/spec/spec.opts +2 -0
  136. data/spec/spec_helper.rb +6 -0
  137. data/yajl-ruby.gemspec +194 -0
  138. metadata +211 -0
@@ -0,0 +1,7 @@
1
+ Makefile
2
+ benchmark/subjects/contacts.*
3
+ *.o
4
+ *.dylib
5
+ *.bundle
6
+ TODO.txt
7
+ tmp/*
@@ -0,0 +1,217 @@
1
+ # Changelog
2
+
3
+ ## 0.6.4 (?)
4
+ * Fixed a bug in Yajl::Encoder which allowed direct, unescaped encoding of NaN, Infinity and -Infinity.
5
+ It will now properly throw a Yajl::EncodeError exception if either of these values are found unescaped.
6
+ * Update bundled Yajl library to 1.0.6
7
+
8
+ ## 0.6.3 (August 25th, 2009)
9
+ * Fixed a bug in the JSON gem compatibility API where strings weren't being properly escaped
10
+
11
+ ## 0.6.2 (August 25th, 2009)
12
+ * Fixed a bug surfaced by an existing library providing a to_json method, and Yajl would double-quote the values provided
13
+
14
+ ## 0.6.1 (August 20th, 2009)
15
+ * Fixed a bug in Yajl::HttpStream where responses contained multiple JSON strings but weren't Transfer-Encoding: chunked (thanks @dacort!)
16
+
17
+ ## 0.6.0 (August 19th, 2009)
18
+ * Added POST, PUT and DELETE support to Yajl::HttpStream
19
+ ** POST support initially contributed by jdg (http://github.com/jdg) - Although oortle (http://github.com/oortle) coded it up in a fork with it as well.
20
+
21
+ ## 0.5.12 (July 31st, 2009)
22
+ * Add another option that can be passed to Yajl::Encoder's constructor (:terminator) to allow the caller some control over
23
+ when a full JSON string has been generated by the encoder. More information on it's use in the README
24
+
25
+ ## 0.5.11 (July 14th, 2009)
26
+ * fixing a bug Aman found with to_json on non-primitive Ruby objects and double-quoting in the JSON compat API
27
+
28
+ ## 0.5.10 (July 13th, 2009)
29
+ * Bugfix for the JSON gem compatibility API's default Object#to_json helper
30
+
31
+ ## 0.5.9 (July 9th, 2009)
32
+ * Bugfix for Yajl::Encoder where encoding a hash like {:a => :b} would get stuck in an infinite loop
33
+
34
+ ## 0.5.8 (July 6th, 2009)
35
+ * Bugfix in Yajl::HttpStream for proper handling of the Content-type header (Rob Sharp)
36
+ * Yajl::Encoder now has an on_progress callback setter, which can be used to harness the encoder's streaming ability.
37
+ ** The passed Proc/lambda will be called, and passed every chunk (currently 8kb) of the encoded JSON string as it's being encoded.
38
+ * API CHANGE WARNING: Yajl::Encoder.encode's block will now be used as (and work the same as) the on_progress callback
39
+ ** This means the block will be passed chunks of the JSON string at a time, giving the caller the ability to start processing the encoded data while it's still being encoded.
40
+ * fixed grammatical error in README (Neil Berkman)
41
+ * Added some encoder examples
42
+
43
+ ## 0.5.7 (June 23rd, 2009)
44
+ * You can now pass parser options (like :symbolize_keys for example) to Yajl::HttpStream.get
45
+ * Refactored spec tests a bit, DRYing up the Yajl::HttpStream specs quite a bit.
46
+ * Added a spec rake task, and spec.opts file
47
+ * Updated and renamed rcov rake task, and added rcov.opts file
48
+
49
+ ## 0.5.6 (June 19th, 2009)
50
+ * Added JSON.default_options hash to the JSON gem compatibility API
51
+ * Split out the JSON gem compatibility API's parsing and encoding methods into individually includable files
52
+ ** the use case here is if you *only* want parsing, or *only* want encoding
53
+ ** also, if you don't include encoding it won't include the #to_json overrides which tend to cause problems in some environments.
54
+ * Removed some large benchmark test files to reduce the size of the packaged gem by 1.5MB!
55
+
56
+ ## 0.5.5 (June 17th, 2009)
57
+ * Introduction of the JSON gem compatibility API
58
+ ** NOTE: this isn't a 1:1 compatibility API, the goal was to be compatible with as many of the projects using the JSON gem as possible - not the JSON gem API itself
59
+ ** the compatibility API must be explicitly enabled by requiring 'yajl/json_gem' in your project
60
+ ** JSON.parse, JSON.generate, and the #to_json instance method extension to ruby's primitive classes are all included
61
+ * Fix Yajl::Encoder to ensure map keys are strings
62
+ * Encoding multiple JSON objects to a single stream doesn't separate by a newline character anymore
63
+ * Yajl::Encoder now checks for the existence of, and will call #to_json on any non-primitive object
64
+
65
+ ## 0.5.4 (June 16th, 2009)
66
+ * Yajl::Parser's :symbolize_keys option now defaults to false
67
+ * remove use of sprintf for a little speed improvement while parsing
68
+
69
+ ## 0.5.3 (June 7th, 2009)
70
+ * The IO parameter for Yajl::Encode#encode is now optional, and accepts a block
71
+ ** it will return the resulting JSON string if no IO is passed to stream to
72
+ ** if a block is passed, it will call and pass it the resulting JSON string
73
+ * Yajl::Parser#parse can now parse from a String as well as an IO
74
+ * Added and updated lot of in-code documentation.
75
+ ** all the C code exposed to Ruby should now have comments
76
+ * Added :symbolize_keys option to the Yajl::Parser class, which defaults to true.
77
+ ** Having this option enabled has shown around an 18% speedup in parsing time according to my benchmarks
78
+
79
+ ## 0.5.2 (May 30th, 2009)
80
+ * Added class helper methods Yajl::Encoder.encode(obj, io) and Yajl::Parser.parse(io)
81
+ * added tests for the above
82
+ * Updated Twitter streaming example to have a less verbose output
83
+ * Patch Yajl so encoding can continue as a stream
84
+ * IE: multiple objects encoded onto the same IO
85
+ * added a test for the above
86
+ * Set the internal read buffer size back down to 8kb by default
87
+ * Added an internal write buffer size (set to 8kb by default) which is used to throttle writes to the output stream
88
+ * This is to fix a major performance bug/issue with the IO#write C method in ruby 1.9.x (I've opened a bug with them about it)
89
+ * Fixed a typo in a one-off parsing spec test
90
+ * Updated benchmarks to work properly in 1.9 (required removal ActiveSupport benchmarking for now)
91
+ * Updated spec tests to respect ridiculous differences in hash key ordering between 1.8 and 1.9
92
+
93
+
94
+ ## 0.5.1 (May 25th, 2009)
95
+ * added some more tests for the new API
96
+ * inlined a couple of hot functions used in parsing for a little speedup
97
+ * updates to readme, reflecting changes in API
98
+ * version bump to push another gem build
99
+
100
+ ## 0.5.0 (May 25th, 2009)
101
+ * Refactored internal API so the caller can specify initialization options for the Parser and Encoder respectively. Two new classes were introduced as a result - Yajl::Parser and Yajl::Encoder. The newly refactored codebase is cleaner, thread-safe and removed all of the hack-code that was trickled around to make things work in the previous implementation. She's much more seaworthy now cap'n!
102
+ * Yajl::Parser.new accepts two options, :allow_comments and :check_utf8 which both default to true
103
+ * Yajl::Encoder.new accepts two options, :pretty and :indent which default to false and " " respectively
104
+ * cleaned up a lot of state code, that to my knowledge prevented yajl-ruby from being used in a thread-safe environment.
105
+ * added deprecated messaging to Yajl::Stream.parse and Yajl::Stream.encode - these will likely go away before 0.6.0
106
+ * fixed a bug in the chunked http response parser regarding partially received chunks
107
+ * added a Twitter Search API example showing off the HttpStream API
108
+
109
+ ## 0.4.9 (May 20th, 2009)
110
+ * fixed some parser state bugs surfaced by edge cases
111
+ * added support for Chunked HTTP response bodies in Yajl::HttpStream
112
+ * added support for passing a block to Yajl::HttpStream.get that will be used as a callback whenever a JSON object is parsed off the stream (even if there is more than one!)
113
+ * added an examples folder, and put an example using the Twitter Streaming API in there to start
114
+ * added some more spec tests, this time around Chunked parsing and continuously parsing multiple JSON strings
115
+
116
+ ## 0.4.8 (May 18th, 2009)
117
+ * fixed a totally bone-head compilation problem, I created for myself ;)
118
+
119
+ ## 0.4.7 (May 18th, 2009)
120
+ * Bundling Yajl sources to remove the need to install them (and CMake) separately (Thank you Lloyd!!!) This means you can now simply install the gem and be off and running
121
+ * Added some spec tests for Yajl::HttpStream
122
+ * Added some spec tests for Yajl::Stream.encode
123
+ * added some more thank you's, where credit's due - in the readme
124
+ * updated the unicode.json file to reflect a "real-life" JSON response
125
+ * reorganized spec tests into their functional areas
126
+ * added an rcov rake task to generate code coverage output
127
+
128
+ ## 0.4.6 (May 17th, 2009)
129
+ * Applied a patch from benburkert (http://github.com/benburkert) to fix HTTP Basic Auth in Yajl::HttpStream.get
130
+
131
+ ## 0.4.5 (May 17th, 2009)
132
+ * added Yajl::Stream.encode(hash, io)
133
+ * generates a JSON string stream, and writes to IO
134
+ * compressed StreamWriter helpers added as well
135
+ * fixed a pretty lame segfault in (x86_64 only?) ubuntu/linux
136
+ * changed the compiled extension to have a more specific name (yajl_ext) for easier loading
137
+ * removed forced-load of .bundle file, for the rest of the planet aside from OSX users
138
+ * added some more benchmarks to compare to other forms of serialization in Ruby
139
+ * various readme updates
140
+
141
+ ## 0.4.4 (May 12th, 2009)
142
+ * NOTE: Breaking API change:
143
+ * renamed Yajl::GzipStreamReader to Yajl::Gzip::StreamReader
144
+ * added Yajl::Bzip2::StreamReader
145
+ * depends on the bzip2-ruby gem if you want to use it, if not Yajl::Bzip2 won't be loaded
146
+ * added Yajl::Deflate::StreamReader
147
+ * actually uses Zlib::Inflate for stream decompression
148
+ * added parse(io) class methods to Yajl::Gzip::StreamReader and Yajl::Bzip2::StreamReader as a helper for parsing compressed streams.
149
+ * updated Yajl::HttpStream to request responses compressed as deflate and bzip2 in addition to gzip
150
+ * fixed a bug regarding parsing Integers as Floats (so 123456 would have be parsed and returned as 123456.0)
151
+ * fixed a bug which caused a segfault in ruby's GC during string replacement in Yajl::Gzip and Yajl::Bzip2's StreamReader#read methods
152
+ * added support for user-specified User-Agent strings in Yajl::HttpStream
153
+
154
+ ## 0.4.3 (May 2nd, 2009)
155
+ * adding text/plain as an allowed mime-type for Yajl::HttpStream for webservers that respond with it instead of application/json (ahem...Yelp...)
156
+ * renamed specs folder to spec for no reason at all
157
+
158
+ ## 0.4.2 (April 30th, 2009)
159
+ * Yajl::HttpStream is now sending "proper" http request headers
160
+ * Yajl::HttpStream will request HTTP-Basic auth if credentials are provided in the passed URI
161
+ * cleanup requires
162
+
163
+ ## 0.4.1 (April 30th, 2009)
164
+ * fixed a typo in the stream.rb benchmark file
165
+ * fixed a bug in Yajl::Stream.parse that was causing "strange" Ruby malloc errors on large files, with large strings
166
+ * added Yajl::GzipStreamReader as a wrapper around Zlib::GzipReader to allow for standard IO#read behavior
167
+ * this allows Yajl::Stream to read off of a Gzip stream directly
168
+
169
+ ## 0.4.0 (April 29th, 2009)
170
+ * NOTE: Breaking API change:
171
+ * refactored Stream parsing methods out of Yajl::Native into Yajl::Stream
172
+ * removed Yajl::Native namespace/module
173
+ * Addition of Yajl::HttpStream module
174
+ * This module is for streaming JSON HTTP responses directly into Yajl (as they're being received) for increased awesomeness
175
+ * it currently supports basic get requests with Yajl::HttpStream.get(uri)
176
+ * it also supports (and prefers) output compressed (gzip) responses
177
+ * Addition Yajl::Chunked module
178
+ * This module is for feeding Yajl JSON pieces at a time, instead of an entire IO object
179
+ * This works very well in environments like an EventMachine app where data is received in chunks by design
180
+ * decreased read buffer for Yajl::Stream from 8kb to 4kb
181
+
182
+ ## 0.3.4 (April 24th, 2009)
183
+ * turned Unicode checks back on in the Yajl parser now that it's fixed (thanks Lloyd!)
184
+ * this also bumps the yajl version dependency requirement to 1.0.4
185
+ * better guessing of Integer/Float from number found instead of just trying to create a BigNum no matter what
186
+ * changed extconf.rb to fail Makefile creation if yajl isn't found
187
+ * added a test to check for parsing Infinity due to a Float overflow
188
+
189
+ ## 0.3.3 (April 24th, 2009)
190
+ * 1.9 compatibility
191
+
192
+ ## 0.3.2 (April 24th, 2009)
193
+ * version bump: forgot to include yajl.c in the gem
194
+
195
+ ## 0.3.1 (April 23rd, 2009)
196
+ * fixed borked gemspec
197
+
198
+ ## 0.3.0 (April 23rd, 2009)
199
+ * slight refactor of ActiveSupport tests to better reflect how they actually exist in ActiveSupport
200
+ * typo correction in the changelog which had the years in 2008
201
+ * added some initial spec tests
202
+ * ported some from ActiveSupport to ensure proper compatibility
203
+ * included 57 JSON fixtures to test against, all of which pass
204
+ * changed parser config to not check for invalid unicode characters as Ruby is going to do this anyway (?). This resolves the remaining test failures around unicode.
205
+ * changed how the parser was dealing with numbers to prevent overflows
206
+ * added an exception class Yajl::ParseError which is now used in place of simply printing to STDERR upon a parsing error
207
+ * renamed a couple of JSON test files in the benchmark folder to better represent their contents
208
+ * misc README updates
209
+
210
+ ## 0.2.1 (April 23rd, 2009)
211
+ * fixed parsing bug - also fixed failing ActiveSupport test failures (except for the unicode one, which is an issue in Yajl itself)
212
+
213
+ ## 0.2.0 (April 22nd, 2009)
214
+ * updated gemspec and README
215
+
216
+ ## 0.1.0 (April 21st, 2009)
217
+ * initial release - gemified
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2008-2009 Brian Lopez - http://github.com/brianmario
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,315 @@
1
+ = YAJL C Bindings for Ruby
2
+
3
+ This gem is a C binding to the excellent YAJL JSON parsing and generation library.
4
+
5
+ You can read more info at the projects website http://lloyd.github.com/yajl or check out it's codes at http://github.com/lloyd/yajl.
6
+
7
+ == Features
8
+
9
+ * JSON parsing and encoding directly to and from an IO stream (file, socket, etc) or String. Compressed stream parsing and encoding supported for Bzip2, Gzip and Deflate.
10
+ * Parse and encode *multiple* JSON objects to and from streams or strings continuously.
11
+ * JSON gem compatibility API - allows yajl-ruby to be used as a drop-in replacement for the JSON gem
12
+ * Basic HTTP client (only GET requests supported for now) which parses JSON directly off the response body *as it's being received*
13
+ * ~3.5x faster than JSON.generate
14
+ * ~1.9x faster than JSON.parse
15
+ * ~4.5x faster than YAML.load
16
+ * ~377.5x faster than YAML.dump
17
+ * ~1.5x faster than Marshal.load
18
+ * ~2x faster than Marshal.dump
19
+
20
+ == How to install
21
+
22
+ First make sure you've got Gemcutter in your sources list:
23
+
24
+ gem sources -a http://gemcutter.org
25
+
26
+ Then go ahead and install it as usual:
27
+
28
+ sudo gem install yajl-ruby
29
+
30
+ == Example of use
31
+
32
+ NOTE: I'm building up a collection of small examples in the examples (http://github.com/brianmario/yajl-ruby/tree/master/examples) folder.
33
+
34
+ First, you're probably gonna want to require it:
35
+
36
+ require 'yajl'
37
+
38
+ === Parsing
39
+
40
+ Then maybe parse some JSON from:
41
+
42
+ a File IO
43
+
44
+ json = File.new('test.json', 'r')
45
+ parser = Yajl::Parser.new
46
+ hash = parser.parse(json)
47
+
48
+ or maybe a StringIO
49
+
50
+ json = StringIO.new("...some JSON...")
51
+ parser = Yajl::Parser.new
52
+ hash = parser.parse(json)
53
+
54
+ or maybe STDIN
55
+
56
+ cat someJsonFile.json | ruby -ryajl -e "puts Yajl::Parser.parse(STDIN).inspect"
57
+
58
+
59
+ Or lets say you didn't have access to the IO object that contained JSON data, but instead
60
+ only had access to chunks of it at a time. No problem!
61
+
62
+ (Assume we're in an EventMachine::Connection instance)
63
+
64
+ def post_init
65
+ @parser = Yajl::Parser.new
66
+ end
67
+
68
+ def object_parsed(obj)
69
+ puts "Sometimes one pays most for the things one gets for nothing. - Albert Einstein"
70
+ puts obj.inspect
71
+ end
72
+
73
+ def connection_completed
74
+ # once a full JSON object has been parsed from the stream
75
+ # object_parsed will be called, and passed the constructed object
76
+ @parser.on_parse_complete = method(:object_parsed)
77
+ end
78
+
79
+ def receive_data(data)
80
+ # continue passing chunks
81
+ @parser << data
82
+ end
83
+
84
+ Or if you don't need to stream it, it'll just return the built object from the parse when it's done.
85
+ NOTE: if there are going to be multiple JSON strings in the input, you *must* specify a block or callback as this
86
+ is how yajl-ruby will hand you (the caller) each object as it's parsed off the input.
87
+
88
+ obj = Yajl::Parser.parse(str_or_io)
89
+
90
+ Or how about a JSON API HTTP request?
91
+ This actually makes a request using a raw TCPSocket, then parses the JSON body right off the socket. While it's being received over the wire!
92
+
93
+ require 'uri'
94
+ require 'yajl/http_stream'
95
+
96
+ url = URI.parse("http://search.twitter.com/search.json?q=engineyard")
97
+ results = Yajl::HttpStream.get(url)
98
+
99
+ Or do the same request, with Gzip and Deflate output compression support (also supports Bzip2, if loaded):
100
+ (this does the same raw socket Request, but transparently parses the compressed response body)
101
+
102
+ require 'uri'
103
+ require 'yajl/gzip'
104
+ require 'yajl/deflate'
105
+ require 'yajl/http_stream'
106
+
107
+ url = URI.parse("http://search.twitter.com/search.json?q=engineyard")
108
+ results = Yajl::HttpStream.get(url)
109
+
110
+ Since yajl-ruby parses JSON as a stream, supporting API's like Twitter's Streaming API are a piece-of-cake.
111
+ You can simply supply a block to Yajl::HttpStream.get, which is used as the callback for when a JSON object has been
112
+ unserialized off the stream. For the case of this Twitter Streaming API call, the callback gets fired a few times a second (depending on your connection speed).
113
+ The code below is all that's needed to make the request and stream unserialized Ruby hashes off the response, continuously.
114
+ You'll note that I've enabled the :symbolize_keys parser option as well. Doing so is much more efficient for parsing JSON streams with
115
+ lots of repetitive keys - for things like result sets or multiple API requests - than the same parse with string keys.
116
+ This is because Ruby will reuse (and never GC) it's symbol table. Be that as it may, if you want to parse JSON strings with random key names
117
+ it's much better to leave string keys enabled (the default), so they can get GC'd later.
118
+
119
+ require 'uri'
120
+ require 'yajl/http_stream'
121
+
122
+ uri = URI.parse("http://#{username}:#{password}@stream.twitter.com/spritzer.json")
123
+ Yajl::HttpStream.get(uri, :symbolize_keys => true) do |hash|
124
+ puts hash.inspect
125
+ end
126
+
127
+ Or how about parsing directly from a compressed file?
128
+
129
+ require 'yajl/bzip2'
130
+
131
+ file = File.new('some.json.bz2', 'r')
132
+ result = Yajl::Bzip2::StreamReader.parse(file)
133
+
134
+ === Encoding
135
+
136
+ Since yajl-ruby does everything using streams, you simply need to pass the object to encode, and the IO to write the stream to (this happens in chunks).
137
+
138
+ This allows you to encode JSON as a stream, writing directly to a socket
139
+
140
+ socket = TCPSocket.new(192.168.1.101, 9000)
141
+ hash = {:foo => 12425125, :bar => "some string", ... }
142
+ encoder = Yajl::Encoder.new
143
+ Yajl::Encoder.encode(hash, socket)
144
+
145
+ Or what if you wanted to compress the stream over the wire?
146
+
147
+ require 'yajl/gzip'
148
+ socket = TCPSocket.new(192.168.1.101, 9000)
149
+ hash = {:foo => 12425125, :bar => "some string", ... }
150
+ Yajl::Gzip::StreamWriter.encode(hash, socket)
151
+
152
+ Or what about encoding multiple objects to JSON over the same stream?
153
+ This example will encode and send 50 JSON objects over the same stream, continuously.
154
+
155
+ socket = TCPSocket.new(192.168.1.101, 9000)
156
+ encoder = Yajl::Encoder.new
157
+ 50.times do
158
+ hash = {:current_time => Time.now.to_f, :foo => 12425125}
159
+ encoder.encode(hash, socket)
160
+ end
161
+
162
+ Using EventMachine and you want to encode and send in chunks?
163
+ (Assume we're in an EventMachine::Connection instance)
164
+
165
+ def post_init
166
+ # Passing a :terminator character will let us determine when the encoder
167
+ # is done encoding the current object
168
+ @encoder = Yajl::Encoder.new(:terminator => nil)
169
+ motd_contents = File.read("/path/to/motd.txt")
170
+ status = File.read("/path/to/huge/status_file.txt")
171
+ @motd = {:motd => motd_contents, :system_status => status}
172
+ end
173
+
174
+ def connection_completed
175
+ # The encoder will do it's best to hand you data in chunks that
176
+ # are around 8kb (but you may see some that are larger)
177
+ #
178
+ # It should be noted that you could have also assigned the _on_progress_ callback
179
+ # much like you can assign the _on_parse_complete_ callback with the parser class.
180
+ # Passing a block (like below) essentially tells the encoder to use that block
181
+ # as the callback normally assigned to _on_progress_.
182
+ #
183
+ # Send our MOTD and status
184
+ @encoder.encode(@motd) do |chunk|
185
+ if chunk.nil? # got our terminator, encoding is done
186
+ close_connection_after_writing
187
+ else
188
+ send_data(chunk)
189
+ end
190
+ end
191
+ end
192
+
193
+ But to make things simple, you might just want to let yajl-ruby do all the hard work for you and just hand back
194
+ a string when it's finished. In that case, just don't provide and IO or block (or assign the on_progress callback).
195
+
196
+ str = Yajl::Encoder.encode(obj)
197
+
198
+ You can also use Yajl::Bzip2::StreamWriter and Yajl::Deflate::StreamWriter. So you can pick whichever fits your CPU/bandwidth sweet-spot.
199
+
200
+ == JSON gem Compatibility API
201
+
202
+ The JSON gem compatibility API isn't enabled by default. You have to explicitly require it like so:
203
+
204
+ require 'yajl/json_gem'
205
+
206
+ That's right, you can just replace "require 'json'" with the line above and you're done!
207
+
208
+ This will require yajl-ruby itself, as well as enable it's JSON gem compatibility API.
209
+
210
+ This includes the following API:
211
+
212
+ JSON.parse, JSON.generate, JSON.pretty_generate, JSON.load, JSON.dump
213
+ and all of the #to_json instance method overrides for Ruby's primitive objects
214
+
215
+ Once the compatibility API is enabled, your existing or new project should work as if the JSON gem itself were being used. Only you'll be using Yajl ;)
216
+
217
+ There are a lot more possibilities that I'd love to see other gems/plugins for someday.
218
+
219
+ Some ideas are:
220
+ * parsing logs in JSON format
221
+ * a Rails plugin - DONE! (http://github.com/technoweenie/yajl-rails)
222
+ * builtin support in Rails 3?
223
+ * Rack middleware (ideally the JSON body could be handed to the parser while it's still being received, this is apparently possible with Unicorn)
224
+ * use with ohai - DONE! (http://github.com/opscode/ohai/commit/f89baccc3b9ab587d23e0b6257f6fedffe223c02)
225
+ * JSON API clients (http://github.com/brianmario/crack, http://github.com/brianmario/freckle-api)
226
+ * Patch Marshal#load and Marshal#dump to use JSON? ;)
227
+ * etc...
228
+
229
+ == Benchmarks
230
+
231
+ After I finished implementation - this library performs close to the same as the current JSON.parse (C gem) does on small/medium files.
232
+
233
+ But on larger files, and higher amounts of iteration, this library was around 2x faster than JSON.parse.
234
+
235
+ The main benefit of this library is in it's memory usage.
236
+ Since it's able to parse the stream in chunks, it's memory requirements are very, very low.
237
+
238
+ Here's what parsing a 2.43MB JSON file off the filesystem 20 times looks like:
239
+
240
+ === Memory Usage
241
+
242
+ ==== Average
243
+
244
+ * Yajl::Parser#parse: 32MB
245
+ * JSON.parse: 54MB
246
+ * ActiveSupport::JSON.decode: 63MB
247
+
248
+ ==== Peak
249
+
250
+ * Yajl::Parser#parse: 32MB
251
+ * JSON.parse: 57MB
252
+ * ActiveSupport::JSON.decode: 67MB
253
+
254
+ === Parse Time
255
+
256
+ * Yajl::Parser#parse: 4.54s
257
+ * JSON.parse: 5.47s
258
+ * ActiveSupport::JSON.decode: 64.42s
259
+
260
+ === Encode Time
261
+
262
+ * Yajl::Encoder#encode: 3.59s
263
+ * JSON#to_json: 6.2s
264
+ * ActiveSupport::JSON.encode: 45.58s
265
+
266
+ === Compared to YAML
267
+
268
+ NOTE: I converted the 2.4MB JSON file to YAML for this test.
269
+
270
+ ==== Parse Time (from their respective formats)
271
+
272
+ * Yajl::Parser#parse: 4.33s
273
+ * JSON.parse: 5.37s
274
+ * YAML.load: 19.47s
275
+
276
+ ==== Encode Time (to their respective formats)
277
+
278
+ * Yajl::Encoder#encode: 3.47s
279
+ * JSON#to_json: 6.6s
280
+ * YAML.dump(obj, io): 1309.93s
281
+
282
+ === Compared to Marshal.load/Marshal.dump
283
+
284
+ NOTE: I converted the 2.4MB JSON file to a Hash and a dump file from Marshal.dump for this test.
285
+
286
+ ==== Parse Time (from their respective formats)
287
+
288
+ * Yajl::Parser#parse: 4.54s
289
+ * JSON.parse: 7.40s
290
+ * Marshal.load: 7s
291
+
292
+ ==== Encode Time (to their respective formats)
293
+
294
+ * Yajl::Encoder#encode: 2.39s
295
+ * JSON#to_json: 8.37s
296
+ * Marshal.dump: 4.66s
297
+
298
+ == Third Party Sources Bundled
299
+
300
+ This project includes code from the BSD licensed yajl project, copyright 2007-2009 Lloyd Hilaiel
301
+
302
+ == Special Thanks
303
+
304
+ For those of you using yajl-ruby out in the wild, please hit me up on Twitter (brianmario) or send me a message here on the Githubs describing the site and how you're using it. I'd love to get a list going!
305
+
306
+ I've had a lot of inspiration, and a lot of help. Thanks to everyone who's been a part of this and those to come!
307
+
308
+ * Lloyd Hilaiel - http://github.com/lloyd - for writing Yajl!!
309
+ * Josh Ferguson - http://github.com/besquared - for peer-pressuring me into getting back into C; it worked ;) Also tons of support over IM
310
+ * Jonathan Novak - http://github.com/cypriss - pointer-hacking help
311
+ * Tom Smith - http://github.com/rtomsmith - pointer-hacking help
312
+ * Rick http://github.com/technoweenie - for making an ActiveSupport patch with support for this library and teasing me that it might go into Rails 3. You sure lit a fire under my ass and I got a ton of work done because of it! :)
313
+ * The entire Github Crew - http://github.com/ - my inspiration, time spent writing this, finding Yajl, So many-MANY other things wouldn't have been possible without this awesome service. I owe you guys some whiskey at Kilowatt.
314
+ * benburkert - http://github.com/benburkert
315
+ * Aman Gupta - http://github.com/tmm1 - tons of suggestions and inspiration for the most recent features, and hopefully more to come ;)