brianmario-yajl-ruby 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.rdoc +7 -1
- data/README.rdoc +18 -18
- data/VERSION.yml +1 -1
- data/benchmark/stream.rb +1 -1
- data/ext/yajl.c +2 -4
- data/ext/yajl.h +1 -1
- data/lib/yajl/http_stream.rb +12 -1
- data/yajl-ruby.gemspec +2 -2
- metadata +2 -2
data/CHANGELOG.rdoc
CHANGED
@@ -1,6 +1,12 @@
|
|
1
1
|
= Changelog
|
2
2
|
|
3
|
-
0.4.
|
3
|
+
0.4.1 (April 30th, 2009)
|
4
|
+
* fixed a typo in the stream.rb benchmark file
|
5
|
+
* fixed a bug in Yajl::Stream.parse that was causing "strange" Ruby malloc errors on large files, with large strings
|
6
|
+
* added Yajl::GzipStreamReader as a wrapper around Zlib::GzipReader to allow for standard IO#read behavior
|
7
|
+
** this allows Yajl::Stream to read off of a Gzip stream directly
|
8
|
+
|
9
|
+
0.4.0 (April 29th, 2009)
|
4
10
|
* NOTE: Breaking API change:
|
5
11
|
** refactored Stream parsing methods out of Yajl::Native into Yajl::Stream
|
6
12
|
** removed Yajl::Native namespace/module
|
data/README.rdoc
CHANGED
@@ -82,37 +82,37 @@ After I finished implementation - this library performs close to the same as the
|
|
82
82
|
But on larger files, and higher amounts of iteration, this library was around 2x faster than JSON.parse.
|
83
83
|
|
84
84
|
The main benefit of this library is in it's memory usage.
|
85
|
-
|
86
85
|
Since it's able to parse the stream in chunks, it's memory requirements are very, very low.
|
87
86
|
|
88
|
-
|
87
|
+
Here's what parsing a 2.43MB JSON file off the filesystem 20 times looks like:
|
89
88
|
|
90
|
-
|
91
|
-
|
92
|
-
|
89
|
+
=== Memory Usage
|
90
|
+
|
91
|
+
==== Average
|
93
92
|
|
94
|
-
|
93
|
+
* Yajl::Stream.parse: 32MB
|
94
|
+
* JSON.parse: 54MB
|
95
|
+
* ActiveSupport::JSON.decode: 63MB
|
95
96
|
|
96
|
-
|
97
|
-
That being said, I'd like to invite you to clone/fork and test the crap out of it! ;)
|
97
|
+
==== Peak
|
98
98
|
|
99
|
-
|
99
|
+
* Yajl::Stream.parse: 32MB
|
100
|
+
* JSON.parse: 57MB
|
101
|
+
* ActiveSupport::JSON.decode: 67MB
|
100
102
|
|
101
|
-
|
102
|
-
Here's a list of things I need to get done to improve the awesomeness of this library.
|
103
|
+
=== Parse Time
|
103
104
|
|
104
|
-
*
|
105
|
-
*
|
106
|
-
*
|
107
|
-
* Change API so it can accept a String or an IO
|
105
|
+
* Yajl::Stream.parse: 4.54s
|
106
|
+
* JSON.parse: 5.47s
|
107
|
+
* ActiveSupport::JSON.decode: 64.42s
|
108
108
|
|
109
109
|
== Special Thanks
|
110
110
|
|
111
111
|
I've had a lot of inspiration, and a lot of help. Thanks to everyone who's been a part of this and those to come!
|
112
112
|
|
113
113
|
* Lloyd Hilaiel (http://github.com/lloyd) - for writing Yajl!!
|
114
|
-
* Josh Ferguson (http://github.com/besquared) - for peer-pressuring me into getting back into C; it worked ;)
|
115
|
-
* Jonathan Novak (http://github.com/cypriss) -
|
116
|
-
* Tom Smith (http://github.com/rtomsmith) -
|
114
|
+
* Josh Ferguson (http://github.com/besquared) - for peer-pressuring me into getting back into C; it worked ;) Also tons of support over IM
|
115
|
+
* Jonathan Novak (http://github.com/cypriss) - pointer-hacking help
|
116
|
+
* Tom Smith (http://github.com/rtomsmith) - pointer-hacking help
|
117
117
|
* Rick (http://github.com/technoweenie) - for making an ActiveSupport patch with support for this library and teasing me that it might go into Rails 3. You sure lit a fire under my ass and I got a ton of work done because of it! :)
|
118
118
|
* The entire Github Crew - my inspiration, time spent writing this, finding Yajl, So many-MANY other things wouldn't have been possible without this awesome service. I owe you guys some whiskey at Kilowatt.
|
data/VERSION.yml
CHANGED
data/benchmark/stream.rb
CHANGED
@@ -16,7 +16,7 @@ times = ARGV[1] ? ARGV[1].to_i : 1
|
|
16
16
|
puts "Starting benchmark parsing #{File.size(filename)} bytes of JSON data #{times} times\n\n"
|
17
17
|
Benchmark.bm { |x|
|
18
18
|
x.report {
|
19
|
-
puts "Yajl::
|
19
|
+
puts "Yajl::Stream.parse (C)"
|
20
20
|
times.times {
|
21
21
|
json.rewind
|
22
22
|
Yajl::Stream.parse(json)
|
data/ext/yajl.c
CHANGED
@@ -57,9 +57,7 @@ static int found_boolean(void * ctx, int boolean) {
|
|
57
57
|
}
|
58
58
|
|
59
59
|
static int found_number(void * ctx, const char * numberVal, unsigned int numberLen) {
|
60
|
-
if (strstr(numberVal, ".") != NULL
|
61
|
-
|| strstr(numberVal, "e") != NULL
|
62
|
-
|| strstr(numberVal, "E") != NULL) {
|
60
|
+
if (strstr(numberVal, ".") != NULL || strstr(numberVal, "e") != NULL || strstr(numberVal, "E") != NULL) {
|
63
61
|
set_static_value(ctx, rb_Float(rb_str_new(numberVal, numberLen)));
|
64
62
|
} else {
|
65
63
|
set_static_value(ctx, rb_Integer(rb_str_new(numberVal, numberLen)));
|
@@ -156,7 +154,7 @@ static VALUE t_parse(VALUE self, VALUE io) {
|
|
156
154
|
|
157
155
|
// now parse from the IO
|
158
156
|
while (rb_funcall(io, intern_eof, 0) == Qfalse) {
|
159
|
-
|
157
|
+
rb_funcall(io, intern_io_read, 2, rbufsize, parsed);
|
160
158
|
|
161
159
|
stat = yajl_parse(streamParser, (const unsigned char *)RSTRING_PTR(parsed), RSTRING_LEN(parsed));
|
162
160
|
|
data/ext/yajl.h
CHANGED
@@ -9,7 +9,7 @@ static ID intern_io_read, intern_eof, intern_respond_to, intern_call;
|
|
9
9
|
static int readBufferSize = READ_BUFSIZE;
|
10
10
|
static yajl_parser_config cfg = {1, 1};
|
11
11
|
|
12
|
-
yajl_handle streamParser
|
12
|
+
yajl_handle streamParser, chunkedParser;
|
13
13
|
VALUE context = Qnil;
|
14
14
|
VALUE parse_complete_callback = Qnil;
|
15
15
|
|
data/lib/yajl/http_stream.rb
CHANGED
@@ -54,7 +54,7 @@ module Yajl
|
|
54
54
|
if response_head[:headers]["Content-Type"].include?(MIME_TYPE)
|
55
55
|
case response_head[:headers]["Content-Encoding"]
|
56
56
|
when "gzip"
|
57
|
-
socket =
|
57
|
+
socket = Yajl::GzipStreamReader.new(socket)
|
58
58
|
end
|
59
59
|
return Yajl::Stream.parse(socket)
|
60
60
|
else
|
@@ -64,4 +64,15 @@ module Yajl
|
|
64
64
|
socket.close
|
65
65
|
end
|
66
66
|
end
|
67
|
+
|
68
|
+
# === Yajl::GzipStreamReader
|
69
|
+
#
|
70
|
+
# This is a wrapper around Zlib::GzipReader to allow it's #read method to adhere
|
71
|
+
# to the IO spec, allowing for two parameters (length, and buffer)
|
72
|
+
class GzipStreamReader < ::Zlib::GzipReader
|
73
|
+
def read(len, buffer=nil)
|
74
|
+
buffer.gsub!(/.*/, '')
|
75
|
+
buffer << super(len)
|
76
|
+
end
|
77
|
+
end
|
67
78
|
end
|
data/yajl-ruby.gemspec
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = %q{yajl-ruby}
|
5
|
-
s.version = "0.4.
|
5
|
+
s.version = "0.4.1"
|
6
6
|
|
7
7
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
8
|
s.authors = ["Brian Lopez"]
|
9
|
-
s.date = %q{2009-04-
|
9
|
+
s.date = %q{2009-04-30}
|
10
10
|
s.email = %q{seniorlopez@gmail.com}
|
11
11
|
s.extensions = ["ext/extconf.rb"]
|
12
12
|
s.extra_rdoc_files = [
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: brianmario-yajl-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brian Lopez
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-04-
|
12
|
+
date: 2009-04-30 00:00:00 -07:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|