grammar 0.5 → 0.8
Sign up to get free protection for your applications and to get access to all the features.
- data/benchmark/json.benchmark.rb +355 -0
- data/benchmark/json.grammar.rb +56 -0
- data/benchmark/json.grammar0_5.rb +57 -0
- data/benchmark/json.ll1.rb +155 -0
- data/benchmark/json.peggy.rb +174 -0
- data/benchmark/json.re.rb +81 -0
- data/lib/grammar.rb +212 -639
- data/lib/grammar/ruby.rb +606 -0
- data/lib/grammar/ruby/code.rb +1030 -0
- data/lib/grammar/ruby0.rb +521 -0
- data/lib/grammar/ruby2cext.rb +19 -0
- data/lib/grammar/rubycall.rb +21 -0
- data/test/advanced.rb +105 -0
- data/test/atoms.rb +77 -0
- data/test/basic.rb +32 -0
- data/test/composite.rb +147 -0
- data/test/molecules.rb +125 -0
- data/test/test_demo.rb +200 -0
- data/test/test_ruby.rb +30 -0
- data/test/test_ruby0.rb +30 -0
- data/test/test_ruby2cext.rb +30 -0
- data/test/test_rubycall.rb +30 -0
- metadata +45 -28
- data/samples/fact.tcl +0 -12
- data/samples/infix2postfix.rb +0 -114
- data/samples/tcl.rb +0 -163
- data/samples/test.infix +0 -4
- data/test/test_grammar.rb +0 -274
@@ -0,0 +1,355 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'benchmark'
|
4
|
+
require 'timeout'
|
5
|
+
require 'rubygems'
|
6
|
+
|
7
|
+
class JSON_Generator
|
8
|
+
attr_accessor(:array, :object)
|
9
|
+
def initialize
|
10
|
+
@string1 = '1'
|
11
|
+
@string2 = '2'
|
12
|
+
@array1 = ['abc : 123', {}, false]
|
13
|
+
@array2 = [1.234e-5, {@string2 => nil}, '']
|
14
|
+
@object1 = {@string1 => '\\\n\t', @string2 => true}
|
15
|
+
@object2 = {@string2 => -789, @string2 => []}
|
16
|
+
end
|
17
|
+
def grow
|
18
|
+
@array1,
|
19
|
+
@array2,
|
20
|
+
@object1,
|
21
|
+
@object2 =
|
22
|
+
@array1.reverse << @object2,
|
23
|
+
@array2.reverse.concat(@array1),
|
24
|
+
@object1.merge({@string1 => @array2}),
|
25
|
+
@object2.merge(@object1)
|
26
|
+
@array2
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def filesize(filename)
|
31
|
+
text = IO.read(filename)
|
32
|
+
text.gsub!(/\#[^\{].*/, '')
|
33
|
+
text.gsub!(/\/\*.*?\*\//, '')
|
34
|
+
text.gsub!(/^\s+/, '')
|
35
|
+
text.gsub!(/\t/, ' ')
|
36
|
+
text.gsub!(/\ +/, ' ')
|
37
|
+
text.gsub!(/\ \n/, "\n")
|
38
|
+
text.gsub!(/\s([\{])\s?/) { $1 }
|
39
|
+
text.gsub!(/([\(])\s?/) { $1 }
|
40
|
+
text.gsub!(/\s([\&\+\-]+|\W\=)\s/) { $1 }
|
41
|
+
text.gsub!(/\s?([\|\<\>\=\,\;]+)\s?/) { $1 }
|
42
|
+
text.gsub!(/\s?([\)])/) { $1 }
|
43
|
+
text.gsub!(/\s?([\}])\ ?/) { $1 }
|
44
|
+
#puts
|
45
|
+
#print text
|
46
|
+
text.size
|
47
|
+
end
|
48
|
+
|
49
|
+
def benchmark(label='', filename=nil)
|
50
|
+
constants = {}
|
51
|
+
self.class.constants.each { |constant| constants[constant] = true }
|
52
|
+
load_path = $LOAD_PATH.clone
|
53
|
+
loaded_features = $LOADED_FEATURES.clone
|
54
|
+
!ARGV[0] or Regexp.new(ARGV[0])=~label or return
|
55
|
+
make = yield
|
56
|
+
limit = (ARGV[1]||1).to_f
|
57
|
+
max_tries = 4
|
58
|
+
unless label.empty?
|
59
|
+
STDERR.print(label)
|
60
|
+
if filename
|
61
|
+
STDERR.print(" ".concat(filesize(filename).to_s))
|
62
|
+
else
|
63
|
+
STDERR.print(" -")
|
64
|
+
end
|
65
|
+
end
|
66
|
+
bandwidth = nil
|
67
|
+
generator = JSON_Generator.new
|
68
|
+
tree0 = nil
|
69
|
+
s = nil
|
70
|
+
tree = nil
|
71
|
+
tries = 0
|
72
|
+
done = false
|
73
|
+
begin
|
74
|
+
GC.start
|
75
|
+
tms = Benchmark.measure {
|
76
|
+
if tries>0
|
77
|
+
tries += 1
|
78
|
+
else
|
79
|
+
tree0 = generator.grow
|
80
|
+
end
|
81
|
+
s = tree0.inspect
|
82
|
+
s.gsub!(/=>/, ':')
|
83
|
+
s.gsub!(/nil/, 'null')
|
84
|
+
}
|
85
|
+
if !make
|
86
|
+
return if tms.utime+tms.stime>limit
|
87
|
+
next
|
88
|
+
end
|
89
|
+
#puts s
|
90
|
+
run = make[s]
|
91
|
+
GC.start
|
92
|
+
GC.start
|
93
|
+
tms = Benchmark.measure { tree = run[] }
|
94
|
+
GC.start
|
95
|
+
raise("#{tree.inspect}!=#{tree0.inspect}") if tree!=tree0
|
96
|
+
tree = nil
|
97
|
+
t = tms.utime+tms.stime
|
98
|
+
bandwidth = s.length/t
|
99
|
+
s = nil
|
100
|
+
GC.start
|
101
|
+
if tries.zero?
|
102
|
+
if t>limit
|
103
|
+
tries = 1
|
104
|
+
else
|
105
|
+
next #if t<(limit/3)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
if bandwidth>=1e9
|
109
|
+
bandwidth /= 1e9
|
110
|
+
suffix = "G"
|
111
|
+
elsif bandwidth>=1e6
|
112
|
+
bandwidth /= 1e6
|
113
|
+
suffix = "M"
|
114
|
+
elsif bandwidth>=1e3
|
115
|
+
bandwidth /= 1e3
|
116
|
+
suffix = "K"
|
117
|
+
else
|
118
|
+
suffix = ""
|
119
|
+
end
|
120
|
+
bandwidth = ("%.3g" % bandwidth).concat(suffix).concat("/s")
|
121
|
+
STDERR.print(" ", bandwidth)
|
122
|
+
end until tries>=max_tries
|
123
|
+
STDERR.print("\n")
|
124
|
+
return bandwidth
|
125
|
+
rescue => error
|
126
|
+
p error
|
127
|
+
ensure
|
128
|
+
#Gem.clear_paths
|
129
|
+
#Gem.loaded_specs.clear
|
130
|
+
self.class.constants.each { |constant|
|
131
|
+
next if constants[constant]
|
132
|
+
#puts("removing #{constant}")
|
133
|
+
self.class.send(:remove_const, constant)
|
134
|
+
}
|
135
|
+
$LOAD_PATH.replace(load_path)
|
136
|
+
$LOADED_FEATURES.replace(loaded_features)
|
137
|
+
end
|
138
|
+
|
139
|
+
require 'strscan'
|
140
|
+
require 'stringio'
|
141
|
+
|
142
|
+
json_pure = '/var/lib/gems/1.8/gems/json-1.1.2/lib/json/pure/parser.rb'
|
143
|
+
json_ext = '/var/lib/gems/1.8/gems/json-1.1.2/ext/json/ext/parser/parser.rl'
|
144
|
+
|
145
|
+
benchmark('') {}
|
146
|
+
|
147
|
+
benchmark('json/ext', json_ext) {
|
148
|
+
gem 'json'
|
149
|
+
require 'json/ext'
|
150
|
+
lambda { |s|
|
151
|
+
JSON::Ext::Parser.new(s).method(:parse)
|
152
|
+
}
|
153
|
+
}
|
154
|
+
|
155
|
+
benchmark('Grammar/Ruby2CExt', 'json.grammar.rb') {
|
156
|
+
$LOAD_PATH << '../lib'
|
157
|
+
require '../lib/grammar'
|
158
|
+
grammar = Grammar.new.instance_eval(IO.read('json.grammar.rb'))
|
159
|
+
Kernel.module_eval { alias_method(:_warn, :warn) }
|
160
|
+
Kernel.module_eval { def warn(*);end }
|
161
|
+
require 'grammar/ruby2cext'
|
162
|
+
Kernel.module_eval { alias_method(:warn, :_warn) }
|
163
|
+
grammar_ruby2cext = Grammar::Ruby2CExt.compile(grammar)
|
164
|
+
lambda { |s|
|
165
|
+
i = StringIO.new(s).method(:getc)
|
166
|
+
out = []
|
167
|
+
lambda {
|
168
|
+
grammar_ruby2cext[out, i]
|
169
|
+
out[0]
|
170
|
+
}
|
171
|
+
}
|
172
|
+
}
|
173
|
+
|
174
|
+
benchmark('hand/RE', 'json.re.rb') {
|
175
|
+
load 'json.re.rb'
|
176
|
+
re = JSON.new
|
177
|
+
lambda { |s|
|
178
|
+
i = StringScanner.new(s)
|
179
|
+
lambda {
|
180
|
+
re.parse(i)
|
181
|
+
}
|
182
|
+
}
|
183
|
+
}
|
184
|
+
|
185
|
+
benchmark('hand/LL1', 'json.ll1.rb') {
|
186
|
+
load 'json.ll1.rb'
|
187
|
+
ll1 = JSON.new
|
188
|
+
lambda { |s|
|
189
|
+
i = StringIO.new(s)
|
190
|
+
lambda {
|
191
|
+
ll1.parse(i)
|
192
|
+
}
|
193
|
+
}
|
194
|
+
}
|
195
|
+
|
196
|
+
benchmark('Grammar/Ruby', 'json.grammar.rb') {
|
197
|
+
$LOAD_PATH << '../lib'
|
198
|
+
require '../lib/grammar'
|
199
|
+
grammar = Grammar.new.instance_eval(IO.read('json.grammar.rb'))
|
200
|
+
require 'grammar/ruby'
|
201
|
+
grammar_ruby = Grammar::Ruby.compile(grammar)
|
202
|
+
lambda { |s|
|
203
|
+
i = StringIO.new(s).method(:getc)
|
204
|
+
out = []
|
205
|
+
lambda {
|
206
|
+
grammar_ruby[out, i]
|
207
|
+
out[0]
|
208
|
+
}
|
209
|
+
}
|
210
|
+
}
|
211
|
+
|
212
|
+
benchmark('RACC/ext', 'json.y') {
|
213
|
+
system('racc json.y -o json.racc.rb')
|
214
|
+
load 'json.racc.rb'
|
215
|
+
Racc::Parser.instance_eval {
|
216
|
+
remove_const(:Racc_Main_Parsing_Routine)
|
217
|
+
remove_const(:Racc_YY_Parse_Method )
|
218
|
+
remove_const(:Racc_Runtime_Type )
|
219
|
+
const_set(:Racc_Main_Parsing_Routine, :_racc_do_parse_c)
|
220
|
+
const_set(:Racc_YY_Parse_Method , :_racc_yyparse_c )
|
221
|
+
const_set(:Racc_Runtime_Type , 'c')
|
222
|
+
}
|
223
|
+
racc = JSON.new
|
224
|
+
lambda { |s|
|
225
|
+
i = StringScanner.new(s)
|
226
|
+
lambda {
|
227
|
+
racc.parse(i)
|
228
|
+
}
|
229
|
+
}
|
230
|
+
}
|
231
|
+
|
232
|
+
benchmark('json/pure', json_pure) {
|
233
|
+
gem 'json'
|
234
|
+
require 'json/pure'
|
235
|
+
lambda { |s|
|
236
|
+
JSON::Pure::Parser.new(s).method(:parse)
|
237
|
+
}
|
238
|
+
}
|
239
|
+
|
240
|
+
benchmark('Grammar/RubyCall', 'json.grammar.rb') {
|
241
|
+
$LOAD_PATH << '../lib'
|
242
|
+
require '../lib/grammar'
|
243
|
+
grammar = Grammar.new.instance_eval(IO.read('json.grammar.rb'))
|
244
|
+
require 'grammar/rubycall'
|
245
|
+
grammar_ruby = Grammar::RubyCall.compile(grammar)
|
246
|
+
lambda { |s|
|
247
|
+
i = StringIO.new(s).method(:getc)
|
248
|
+
out = []
|
249
|
+
lambda {
|
250
|
+
grammar_ruby[out, i]
|
251
|
+
out[0]
|
252
|
+
}
|
253
|
+
}
|
254
|
+
}
|
255
|
+
|
256
|
+
benchmark('Grammar/0.5', 'json.grammar0_5.rb') {
|
257
|
+
gem 'grammar'
|
258
|
+
require 'grammar'
|
259
|
+
require 'duck'
|
260
|
+
require 'json.grammar0_5'
|
261
|
+
gem 'cursor'
|
262
|
+
require 'cursor/io'
|
263
|
+
grammar0_5 = JSON.new
|
264
|
+
lambda { |s|
|
265
|
+
i = Cursor::IO.new(StringIO.new(s))
|
266
|
+
lambda {
|
267
|
+
grammar0_5.scan(i, out=[])
|
268
|
+
out[0]
|
269
|
+
}
|
270
|
+
}
|
271
|
+
}
|
272
|
+
|
273
|
+
benchmark('RACC/ruby', 'json.y') {
|
274
|
+
system('racc json.y -o json.racc.rb')
|
275
|
+
load 'json.racc.rb'
|
276
|
+
Racc::Parser.instance_eval {
|
277
|
+
remove_const(:Racc_Main_Parsing_Routine)
|
278
|
+
remove_const(:Racc_YY_Parse_Method )
|
279
|
+
remove_const(:Racc_Runtime_Type )
|
280
|
+
const_set(:Racc_Main_Parsing_Routine, :_racc_do_parse_rb)
|
281
|
+
const_set(:Racc_YY_Parse_Method , :_racc_yyparse_rb )
|
282
|
+
const_set(:Racc_Runtime_Type , 'ruby')
|
283
|
+
}
|
284
|
+
racc = JSON.new
|
285
|
+
lambda { |s|
|
286
|
+
i = StringScanner.new(s)
|
287
|
+
lambda {
|
288
|
+
racc.parse(i)
|
289
|
+
}
|
290
|
+
}
|
291
|
+
}
|
292
|
+
|
293
|
+
benchmark('Treetop', 'json.treetop') {
|
294
|
+
gem 'treetop'
|
295
|
+
require 'treetop'
|
296
|
+
Treetop.load('json.treetop')
|
297
|
+
treetop = JSONParser.new
|
298
|
+
lambda { |s|
|
299
|
+
lambda {
|
300
|
+
treetop.parse(s).obj
|
301
|
+
}
|
302
|
+
}
|
303
|
+
}
|
304
|
+
|
305
|
+
benchmark('GhostWheel', 'json.ghostwheel') {
|
306
|
+
gem 'ghostwheel'
|
307
|
+
require 'ghost_wheel'
|
308
|
+
ghostwheel = GhostWheel.build_parser(IO.read("json.ghostwheel"))
|
309
|
+
lambda { |s|
|
310
|
+
lambda {
|
311
|
+
ghostwheel.parse(s)
|
312
|
+
}
|
313
|
+
}
|
314
|
+
}
|
315
|
+
|
316
|
+
benchmark('Grammar/Ruby0', 'json.grammar.rb') {
|
317
|
+
$LOAD_PATH << '../lib'
|
318
|
+
require '../lib/grammar'
|
319
|
+
grammar = Grammar.new.instance_eval(IO.read('json.grammar.rb'))
|
320
|
+
require 'grammar/ruby0'
|
321
|
+
grammar_ruby0 = Grammar::Ruby0.compile(grammar)
|
322
|
+
lambda { |s|
|
323
|
+
i = StringIO.new(s).method(:getc)
|
324
|
+
out = []
|
325
|
+
lambda {
|
326
|
+
grammar_ruby0[out, i]
|
327
|
+
out[0]
|
328
|
+
}
|
329
|
+
}
|
330
|
+
}
|
331
|
+
|
332
|
+
benchmark('Peggy', 'json.peggy.rb') {
|
333
|
+
gem 'peggy'
|
334
|
+
require 'parser'
|
335
|
+
require 'builder'
|
336
|
+
require 'json.peggy'
|
337
|
+
peggy = JSON.new
|
338
|
+
lambda { |s|
|
339
|
+
peggy.source_text = s
|
340
|
+
lambda {
|
341
|
+
peggy.parse?(:value)
|
342
|
+
peggy.to_ruby
|
343
|
+
}
|
344
|
+
}
|
345
|
+
}
|
346
|
+
|
347
|
+
benchmark('json/ext', json_ext) {
|
348
|
+
gem 'json'
|
349
|
+
require 'json/ext'
|
350
|
+
lambda { |s|
|
351
|
+
JSON::Ext::Parser.new(s).method(:parse)
|
352
|
+
}
|
353
|
+
}
|
354
|
+
|
355
|
+
|
@@ -0,0 +1,56 @@
|
|
1
|
+
Common { |e|
|
2
|
+
ws = (E(?\s) | E(?\t) | E(?\n) | E(?\r)).discard.repeat0
|
3
|
+
|
4
|
+
digit = E(?0..?9)
|
5
|
+
digits = digit.repeat1
|
6
|
+
exp = ((E(?e)|E(?E)) + (E(?+)|E(?-)|NULL)) + digits
|
7
|
+
int0 = E(?0) | E(?1..?9) + digit.repeat0
|
8
|
+
number = ((E(?-) + int0 | int0) + (
|
9
|
+
(E(?.) + digits + (exp|NULL) | exp) + Output { |s| s.to_f } |
|
10
|
+
Output { |s| s.to_i }
|
11
|
+
)).group("")
|
12
|
+
|
13
|
+
hex = digit | E(?a..?f) | E(?A..?F)
|
14
|
+
char = E(?\\).discard + (
|
15
|
+
E(?\") |
|
16
|
+
E(?\\) |
|
17
|
+
E(?\/) |
|
18
|
+
E(?b).discard { e[?\b] } |
|
19
|
+
E(?f).discard { e[?\f] } |
|
20
|
+
E(?n).discard { e[?\n] } |
|
21
|
+
E(?r).discard { e[?\r] } |
|
22
|
+
E(?t).discard { e[?\t] } |
|
23
|
+
E(?u).discard + (hex+hex+hex+hex).group("") { |s|
|
24
|
+
s.to_i(e[16])
|
25
|
+
}
|
26
|
+
) |
|
27
|
+
ANY
|
28
|
+
string = E(?\").discard + char.repeat0(E(?\").discard).group("")
|
29
|
+
|
30
|
+
ws + Recurse { |value|
|
31
|
+
elements = Recurse { |elements|
|
32
|
+
value + ws + (E(?,).discard + ws + elements | NULL)
|
33
|
+
}
|
34
|
+
array = E(?[).discard + ws + (elements + ws|NULL).group([]) +
|
35
|
+
E(?]).discard
|
36
|
+
|
37
|
+
pair = string + ws + E(?:).discard + ws + value
|
38
|
+
members = Recurse { |members|
|
39
|
+
pair + ws + (E(?,).discard + ws + members | NULL)
|
40
|
+
}
|
41
|
+
object = E(?{).discard + ws + (members|NULL).redirect([]) { |buf|
|
42
|
+
e << e.send_splat(e[Hash], :[], buf)
|
43
|
+
} + E(?}).discard
|
44
|
+
|
45
|
+
string |
|
46
|
+
object |
|
47
|
+
array |
|
48
|
+
Chain("true").discard { e[true] } |
|
49
|
+
Chain("false").discard { e[false] } |
|
50
|
+
Chain("null").discard { e[nil] } |
|
51
|
+
number
|
52
|
+
|
53
|
+
} + ws + EOF
|
54
|
+
}
|
55
|
+
|
56
|
+
|
@@ -0,0 +1,57 @@
|
|
1
|
+
class JSON < Grammar
|
2
|
+
|
3
|
+
E = Element
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
|
7
|
+
ws = (E[?\s]|E[?\t]|E[?\n]|E[?\r]).discard.list0
|
8
|
+
|
9
|
+
digit = E[(?0..?9).duck!(:==,:===)]
|
10
|
+
digits = digit.list1
|
11
|
+
exp = (E[?e]|E[?E]) + (E[?+]|E[?-]|NULL) + digits
|
12
|
+
int0 = E[?0] | E[(?1..?9).duck!(:==,:===)] + digit.list0
|
13
|
+
number = ((E[?-] + int0 | int0).group(String) +
|
14
|
+
(E[?.] + digits + (exp|NULL) | exp | NULL).group(String)).
|
15
|
+
filter(Array) { |n| n[1].empty? ? [n[0].to_i] : [n.to_s.to_f] }
|
16
|
+
|
17
|
+
hex = digit | E[(?a..?f).duck!(:==,:===)] | E[(?A..?F).duck!(:==,:===)]
|
18
|
+
char = E[?\\].discard + (
|
19
|
+
E[?\"].filter { "\"" } |
|
20
|
+
E[?\\].filter { "\\" } |
|
21
|
+
E[?\/].filter { "\/" } |
|
22
|
+
E[?b].filter { "\b" } |
|
23
|
+
E[?f].filter { "\f" } |
|
24
|
+
E[?n].filter { "\n" } |
|
25
|
+
E[?r].filter { "\r" } |
|
26
|
+
E[?t].filter { "\t" } |
|
27
|
+
E[?u].discard + (hex+hex+hex+hex).filter { |s| "" << s.to_i(16) }
|
28
|
+
) |
|
29
|
+
ANY
|
30
|
+
string = E[?\"].discard + char.list0(E[?\"].discard).group(String)
|
31
|
+
|
32
|
+
super(ws + Grammar.new { |value|
|
33
|
+
elements = (value + ws).list1(nil, E[?,].discard + ws)
|
34
|
+
array = E[?[].discard + ws + (elements + ws|NULL).group(Array) +
|
35
|
+
E[?]].discard
|
36
|
+
|
37
|
+
pair = string + ws + E[?:].discard + ws + value
|
38
|
+
members = (pair + ws).list1(nil, E[?,].discard + ws)
|
39
|
+
object = E[?{].discard + ws + (members|NULL).filter(Array) { |a|
|
40
|
+
[Hash[*a]]
|
41
|
+
} + E[?}].discard
|
42
|
+
|
43
|
+
string |
|
44
|
+
object |
|
45
|
+
array |
|
46
|
+
(E[?t]+E[?r]+E[?u]+E[?e]).filter { [true] } |
|
47
|
+
(E[?f]+E[?a]+E[?l]+E[?s]+E[?e]).filter { [false] } |
|
48
|
+
(E[?n]+E[?u]+E[?l]+E[?l]).filter { [nil] } |
|
49
|
+
number
|
50
|
+
|
51
|
+
} + ws + EOF)
|
52
|
+
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
|
57
|
+
|