jsgf 0 → 0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +4 -0
- data/README.md +1 -1
- data/Rakefile +17 -1
- data/jsgf.gemspec +2 -2
- data/lib/jsgf/alternation.rb +25 -0
- data/lib/jsgf/grammar.rb +23 -0
- data/lib/jsgf/optional.rb +11 -0
- data/lib/jsgf/parser.rb +390 -0
- data/lib/jsgf/repetition.rb +10 -0
- data/lib/jsgf/tokenizer.rb +35 -0
- data/lib/jsgf.rb +18 -2
- data/test/jsgf/parser.rb +109 -0
- data/test/jsgf/tokenizer.rb +36 -0
- metadata +13 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 66ac3ad9f73f24c8c44884964606916ad2495680
|
4
|
+
data.tar.gz: 1a67637a02d934eee7ee0c53f8586217f68a6567
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6b42838dcefd7eb536f0dd61a9c6ca352fba12b60577960ef316c35dda5d9ce0e681a34ba8b2dfcd70d11b6981f6c22a70700a6cdf806dfd538671df35789a73
|
7
|
+
data.tar.gz: c04356733209a868672753df49626aa595f9b56c91e2a3dfd95493c75599df244ac3b4d93c09f94a54f2e582c51895f02396ec120605e9234830941868534953
|
data/Gemfile
CHANGED
data/README.md
CHANGED
data/Rakefile
CHANGED
@@ -1,2 +1,18 @@
|
|
1
|
-
require
|
1
|
+
require 'bundler/gem_tasks'
|
2
|
+
require 'rake/testtask'
|
2
3
|
|
4
|
+
rule '.rb' => '.y' do |t|
|
5
|
+
`racc -l -o #{t.name} #{t.source}`
|
6
|
+
end
|
7
|
+
|
8
|
+
task :default => :test
|
9
|
+
task compile: 'lib/jsgf/parser.rb'
|
10
|
+
|
11
|
+
# Recompile the parser before building the gem package
|
12
|
+
task :build => :compile
|
13
|
+
|
14
|
+
Rake::TestTask.new(:test => :compile) do |t|
|
15
|
+
t.libs.push "lib"
|
16
|
+
t.test_files = FileList['test/**/*.rb']
|
17
|
+
t.verbose = true
|
18
|
+
end
|
data/jsgf.gemspec
CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "jsgf"
|
7
|
-
spec.version = '0'
|
7
|
+
spec.version = '0.1'
|
8
8
|
spec.authors = ["Brandon Fosdick"]
|
9
9
|
spec.email = ["bfoz@bfoz.net"]
|
10
10
|
spec.summary = %q{Java Speech Grammar Format}
|
@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
|
|
12
12
|
spec.homepage = "http://github.com/bfoz/jsgf-ruby"
|
13
13
|
spec.license = "BSD"
|
14
14
|
|
15
|
-
spec.files = `git ls-files -z`.split("\x0")
|
15
|
+
spec.files = `git ls-files -z`.split("\x0").push('lib/jsgf/parser.rb').reject {|f| File.basename(f) == 'parser.y'}
|
16
16
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
17
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
18
|
spec.require_paths = ["lib"]
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module JSGF
|
2
|
+
class Alternation
|
3
|
+
attr_reader :elements
|
4
|
+
attr_writer :optional
|
5
|
+
attr_reader :tags
|
6
|
+
|
7
|
+
def initialize(*args)
|
8
|
+
@elements = args
|
9
|
+
@optional = false
|
10
|
+
@tags = []
|
11
|
+
end
|
12
|
+
|
13
|
+
def push(*args)
|
14
|
+
@elements.push *args
|
15
|
+
end
|
16
|
+
|
17
|
+
def size
|
18
|
+
@elements.size
|
19
|
+
end
|
20
|
+
|
21
|
+
def weights
|
22
|
+
@elements.map {|a| a[:weight]}
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
data/lib/jsgf/grammar.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
module JSGF
|
2
|
+
class Grammar
|
3
|
+
attr_reader :character_encoding
|
4
|
+
attr_reader :locale
|
5
|
+
attr_reader :private_rules
|
6
|
+
attr_reader :public_rules
|
7
|
+
attr_reader :version
|
8
|
+
|
9
|
+
def initialize(character_encoding:nil, locale:nil, private_rules:{}, public_rules:{}, version:nil)
|
10
|
+
@character_encoding = character_encoding
|
11
|
+
@locale = locale
|
12
|
+
@private_rules = private_rules
|
13
|
+
@public_rules = public_rules
|
14
|
+
@version = version
|
15
|
+
end
|
16
|
+
|
17
|
+
# @!attribute rules
|
18
|
+
# @return [Hash] Returns a {Hash} of the public and private rules, keyed by rule name
|
19
|
+
def rules
|
20
|
+
@public_rules.merge(@private_rules)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/jsgf/parser.rb
ADDED
@@ -0,0 +1,390 @@
|
|
1
|
+
#
|
2
|
+
# DO NOT MODIFY!!!!
|
3
|
+
# This file is automatically generated by Racc 1.4.12
|
4
|
+
# from Racc grammer file "".
|
5
|
+
#
|
6
|
+
|
7
|
+
require 'racc/parser.rb'
|
8
|
+
|
9
|
+
|
10
|
+
require_relative 'alternation'
|
11
|
+
require_relative 'grammar'
|
12
|
+
require_relative 'optional'
|
13
|
+
require_relative 'repetition'
|
14
|
+
|
15
|
+
module JSGF
|
16
|
+
class Parser < Racc::Parser
|
17
|
+
|
18
|
+
|
19
|
+
attr_reader :grammar_name
|
20
|
+
attr_reader :handler
|
21
|
+
|
22
|
+
def initialize(tokenizer)
|
23
|
+
@private_rules = {}
|
24
|
+
@public_rules = {}
|
25
|
+
tokenizer = JSGF::Tokenizer.new(tokenizer) if tokenizer.is_a?(String)
|
26
|
+
@tokenizer = tokenizer
|
27
|
+
super()
|
28
|
+
end
|
29
|
+
|
30
|
+
def define_alternation(*args)
|
31
|
+
if args.first.is_a?(JSGF::Alternation)
|
32
|
+
args.first.push *(args.drop(2))
|
33
|
+
args.first
|
34
|
+
else
|
35
|
+
JSGF::Alternation.new(*(args.each_slice(2).map {|a,b| a}))
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def define_optional(*args)
|
40
|
+
if args.first.respond_to?(:optional)
|
41
|
+
args.first.optional = true
|
42
|
+
args.first
|
43
|
+
else
|
44
|
+
JSGF::Optional.new(*args)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def define_atom(atom, weight=1.0, tags=[])
|
49
|
+
{atom:atom, weight:weight, tags:[]}
|
50
|
+
end
|
51
|
+
|
52
|
+
def define_rule(name, visibility=:private, *args)
|
53
|
+
r = {name:name, visibility:visibility, atoms:args}
|
54
|
+
if visibility == :private
|
55
|
+
@private_rules[name] = r
|
56
|
+
else
|
57
|
+
@public_rules[name] = r
|
58
|
+
end
|
59
|
+
r
|
60
|
+
end
|
61
|
+
|
62
|
+
def next_token
|
63
|
+
@tokenizer.next_token
|
64
|
+
end
|
65
|
+
|
66
|
+
def parse
|
67
|
+
do_parse
|
68
|
+
JSGF::Grammar.new( character_encoding:@charset,
|
69
|
+
locale:@locale,
|
70
|
+
private_rules:@private_rules,
|
71
|
+
public_rules:@public_rules,
|
72
|
+
version:@version)
|
73
|
+
end
|
74
|
+
##### State transition tables begin ###
|
75
|
+
|
76
|
+
racc_action_table = [
|
77
|
+
40, 37, 17, 11, 16, 40, 37, 38, 11, 39,
|
78
|
+
40, 37, 38, 11, 39, 40, 44, 38, 11, 39,
|
79
|
+
40, 37, 38, 11, 39, 40, 37, 38, 11, 39,
|
80
|
+
40, 37, 38, 11, 39, 40, 37, 38, 11, 39,
|
81
|
+
9, 13, 38, 45, 39, 13, 11, 9, 13, 13,
|
82
|
+
11, 49, 48, 11, 11, 61, 49, 62, 49, 51,
|
83
|
+
60, 49, 28, 47, 27, 46, 52, 53, 52, 53,
|
84
|
+
30, 29, 26, 11, 24, 58, 23, 51, 11, 18,
|
85
|
+
15, 5, 4 ]
|
86
|
+
|
87
|
+
racc_action_check = [
|
88
|
+
59, 59, 4, 59, 4, 32, 32, 59, 32, 59,
|
89
|
+
24, 24, 32, 24, 32, 37, 25, 24, 37, 24,
|
90
|
+
38, 38, 37, 38, 37, 39, 39, 38, 39, 38,
|
91
|
+
49, 49, 39, 49, 39, 44, 44, 49, 44, 49,
|
92
|
+
2, 2, 44, 26, 44, 6, 2, 7, 7, 20,
|
93
|
+
6, 56, 31, 7, 20, 56, 31, 57, 55, 33,
|
94
|
+
55, 57, 17, 28, 17, 28, 35, 35, 54, 54,
|
95
|
+
23, 22, 15, 13, 12, 47, 11, 50, 9, 5,
|
96
|
+
3, 1, 0 ]
|
97
|
+
|
98
|
+
racc_action_pointer = [
|
99
|
+
79, 81, 36, 78, -5, 79, 40, 43, nil, 68,
|
100
|
+
nil, 69, 62, 63, nil, 65, nil, 55, nil, nil,
|
101
|
+
44, nil, 62, 59, 3, 4, 34, nil, 56, nil,
|
102
|
+
nil, 43, -2, 53, nil, 48, nil, 8, 13, 18,
|
103
|
+
nil, nil, nil, nil, 28, nil, nil, 66, nil, 23,
|
104
|
+
71, nil, nil, nil, 50, 45, 38, 48, nil, -7,
|
105
|
+
nil, nil, nil ]
|
106
|
+
|
107
|
+
racc_action_default = [
|
108
|
+
-35, -35, -1, -35, -35, -35, -2, -35, -10, -35,
|
109
|
+
-13, -35, -35, -35, -4, -35, -5, -35, 63, -14,
|
110
|
+
-3, -11, -35, -35, -35, -35, -35, -6, -35, -12,
|
111
|
+
-15, -35, -18, -20, -22, -24, -25, -35, -35, -35,
|
112
|
+
-29, -30, -31, -32, -35, -9, -7, -35, -16, -35,
|
113
|
+
-21, -23, -33, -34, -26, -35, -35, -35, -8, -19,
|
114
|
+
-27, -28, -17 ]
|
115
|
+
|
116
|
+
racc_goto_table = [
|
117
|
+
50, 31, 12, 19, 1, 6, 12, 12, 14, 22,
|
118
|
+
20, 3, 7, 25, 8, 55, 56, 19, 59, 21,
|
119
|
+
12, 57, 2, 54, nil, nil, nil, 50 ]
|
120
|
+
|
121
|
+
racc_goto_check = [
|
122
|
+
12, 10, 8, 9, 1, 3, 8, 8, 6, 8,
|
123
|
+
3, 5, 4, 8, 7, 10, 10, 9, 11, 7,
|
124
|
+
8, 10, 2, 14, nil, nil, nil, 12 ]
|
125
|
+
|
126
|
+
racc_goto_pointer = [
|
127
|
+
nil, 4, 22, 3, 10, 11, 5, 12, 0, -3,
|
128
|
+
-23, -31, -32, nil, -14, nil, nil, nil ]
|
129
|
+
|
130
|
+
racc_goto_default = [
|
131
|
+
nil, nil, nil, nil, nil, nil, nil, nil, 41, 10,
|
132
|
+
nil, 32, 33, 34, 35, 36, 42, 43 ]
|
133
|
+
|
134
|
+
racc_reduce_table = [
|
135
|
+
0, 0, :racc_error,
|
136
|
+
1, 21, :_reduce_none,
|
137
|
+
2, 21, :_reduce_none,
|
138
|
+
3, 21, :_reduce_none,
|
139
|
+
2, 22, :_reduce_none,
|
140
|
+
2, 25, :_reduce_none,
|
141
|
+
3, 25, :_reduce_6,
|
142
|
+
4, 25, :_reduce_7,
|
143
|
+
5, 25, :_reduce_8,
|
144
|
+
3, 26, :_reduce_9,
|
145
|
+
1, 24, :_reduce_none,
|
146
|
+
2, 24, :_reduce_none,
|
147
|
+
3, 27, :_reduce_none,
|
148
|
+
1, 23, :_reduce_none,
|
149
|
+
2, 23, :_reduce_none,
|
150
|
+
3, 28, :_reduce_15,
|
151
|
+
4, 29, :_reduce_16,
|
152
|
+
5, 29, :_reduce_17,
|
153
|
+
1, 30, :_reduce_18,
|
154
|
+
3, 30, :_reduce_19,
|
155
|
+
1, 31, :_reduce_20,
|
156
|
+
2, 31, :_reduce_21,
|
157
|
+
1, 32, :_reduce_none,
|
158
|
+
2, 32, :_reduce_23,
|
159
|
+
1, 33, :_reduce_none,
|
160
|
+
1, 33, :_reduce_none,
|
161
|
+
2, 35, :_reduce_26,
|
162
|
+
3, 36, :_reduce_27,
|
163
|
+
3, 37, :_reduce_28,
|
164
|
+
1, 34, :_reduce_29,
|
165
|
+
1, 34, :_reduce_none,
|
166
|
+
1, 34, :_reduce_none,
|
167
|
+
1, 34, :_reduce_none,
|
168
|
+
2, 34, :_reduce_33,
|
169
|
+
2, 34, :_reduce_34 ]
|
170
|
+
|
171
|
+
racc_reduce_n = 35
|
172
|
+
|
173
|
+
racc_shift_n = 63
|
174
|
+
|
175
|
+
racc_token_table = {
|
176
|
+
false => 0,
|
177
|
+
:error => 1,
|
178
|
+
:GRAMMAR => 2,
|
179
|
+
:HEADER => 3,
|
180
|
+
:IMPORT => 4,
|
181
|
+
:PUBLIC => 5,
|
182
|
+
:TAG => 6,
|
183
|
+
:TOKEN => 7,
|
184
|
+
:WEIGHT => 8,
|
185
|
+
";" => 9,
|
186
|
+
"<" => 10,
|
187
|
+
">" => 11,
|
188
|
+
"=" => 12,
|
189
|
+
"|" => 13,
|
190
|
+
"(" => 14,
|
191
|
+
")" => 15,
|
192
|
+
"[" => 16,
|
193
|
+
"]" => 17,
|
194
|
+
"*" => 18,
|
195
|
+
"+" => 19 }
|
196
|
+
|
197
|
+
racc_nt_base = 20
|
198
|
+
|
199
|
+
racc_use_result_var = true
|
200
|
+
|
201
|
+
Racc_arg = [
|
202
|
+
racc_action_table,
|
203
|
+
racc_action_check,
|
204
|
+
racc_action_default,
|
205
|
+
racc_action_pointer,
|
206
|
+
racc_goto_table,
|
207
|
+
racc_goto_check,
|
208
|
+
racc_goto_default,
|
209
|
+
racc_goto_pointer,
|
210
|
+
racc_nt_base,
|
211
|
+
racc_reduce_table,
|
212
|
+
racc_token_table,
|
213
|
+
racc_shift_n,
|
214
|
+
racc_reduce_n,
|
215
|
+
racc_use_result_var ]
|
216
|
+
|
217
|
+
Racc_token_to_s_table = [
|
218
|
+
"$end",
|
219
|
+
"error",
|
220
|
+
"GRAMMAR",
|
221
|
+
"HEADER",
|
222
|
+
"IMPORT",
|
223
|
+
"PUBLIC",
|
224
|
+
"TAG",
|
225
|
+
"TOKEN",
|
226
|
+
"WEIGHT",
|
227
|
+
"\";\"",
|
228
|
+
"\"<\"",
|
229
|
+
"\">\"",
|
230
|
+
"\"=\"",
|
231
|
+
"\"|\"",
|
232
|
+
"\"(\"",
|
233
|
+
"\")\"",
|
234
|
+
"\"[\"",
|
235
|
+
"\"]\"",
|
236
|
+
"\"*\"",
|
237
|
+
"\"+\"",
|
238
|
+
"$start",
|
239
|
+
"grammar",
|
240
|
+
"header",
|
241
|
+
"rule_list",
|
242
|
+
"import_header",
|
243
|
+
"jsgf_header",
|
244
|
+
"grammar_header",
|
245
|
+
"import_statement",
|
246
|
+
"rule_name",
|
247
|
+
"rule",
|
248
|
+
"alternate_list",
|
249
|
+
"rule_expansion",
|
250
|
+
"tagged_rule_item",
|
251
|
+
"rule_item",
|
252
|
+
"rule_atom",
|
253
|
+
"weighted_item",
|
254
|
+
"rule_group",
|
255
|
+
"rule_optional" ]
|
256
|
+
|
257
|
+
Racc_debug_parser = false
|
258
|
+
|
259
|
+
##### State transition tables end #####
|
260
|
+
|
261
|
+
# reduce 0 omitted
|
262
|
+
|
263
|
+
# reduce 1 omitted
|
264
|
+
|
265
|
+
# reduce 2 omitted
|
266
|
+
|
267
|
+
# reduce 3 omitted
|
268
|
+
|
269
|
+
# reduce 4 omitted
|
270
|
+
|
271
|
+
# reduce 5 omitted
|
272
|
+
|
273
|
+
def _reduce_6(val, _values, result)
|
274
|
+
@version = val[1];
|
275
|
+
result
|
276
|
+
end
|
277
|
+
|
278
|
+
def _reduce_7(val, _values, result)
|
279
|
+
@version = val[1]; @charset = val[2];
|
280
|
+
result
|
281
|
+
end
|
282
|
+
|
283
|
+
def _reduce_8(val, _values, result)
|
284
|
+
@version = val[1]; @charset = val[2]; @locale = val[3];
|
285
|
+
result
|
286
|
+
end
|
287
|
+
|
288
|
+
def _reduce_9(val, _values, result)
|
289
|
+
@grammar_name = val[1]
|
290
|
+
result
|
291
|
+
end
|
292
|
+
|
293
|
+
# reduce 10 omitted
|
294
|
+
|
295
|
+
# reduce 11 omitted
|
296
|
+
|
297
|
+
# reduce 12 omitted
|
298
|
+
|
299
|
+
# reduce 13 omitted
|
300
|
+
|
301
|
+
# reduce 14 omitted
|
302
|
+
|
303
|
+
def _reduce_15(val, _values, result)
|
304
|
+
result = val[1]
|
305
|
+
result
|
306
|
+
end
|
307
|
+
|
308
|
+
def _reduce_16(val, _values, result)
|
309
|
+
define_rule(val[0], :private, *(val[2..-2]))
|
310
|
+
result
|
311
|
+
end
|
312
|
+
|
313
|
+
def _reduce_17(val, _values, result)
|
314
|
+
define_rule(val[1], :public, val[3])
|
315
|
+
result
|
316
|
+
end
|
317
|
+
|
318
|
+
def _reduce_18(val, _values, result)
|
319
|
+
result = val.first
|
320
|
+
result
|
321
|
+
end
|
322
|
+
|
323
|
+
def _reduce_19(val, _values, result)
|
324
|
+
result = define_alternation(*val)
|
325
|
+
result
|
326
|
+
end
|
327
|
+
|
328
|
+
def _reduce_20(val, _values, result)
|
329
|
+
result = val.first
|
330
|
+
result
|
331
|
+
end
|
332
|
+
|
333
|
+
def _reduce_21(val, _values, result)
|
334
|
+
result = val
|
335
|
+
result
|
336
|
+
end
|
337
|
+
|
338
|
+
# reduce 22 omitted
|
339
|
+
|
340
|
+
def _reduce_23(val, _values, result)
|
341
|
+
val[0][:tags].push(val[1]); result = val[0]
|
342
|
+
result
|
343
|
+
end
|
344
|
+
|
345
|
+
# reduce 24 omitted
|
346
|
+
|
347
|
+
# reduce 25 omitted
|
348
|
+
|
349
|
+
def _reduce_26(val, _values, result)
|
350
|
+
val[1][:weight] = val.first[1..-2].to_f; result = val[1]
|
351
|
+
result
|
352
|
+
end
|
353
|
+
|
354
|
+
def _reduce_27(val, _values, result)
|
355
|
+
result = val[1]
|
356
|
+
result
|
357
|
+
end
|
358
|
+
|
359
|
+
def _reduce_28(val, _values, result)
|
360
|
+
result = define_optional(val[1])
|
361
|
+
result
|
362
|
+
end
|
363
|
+
|
364
|
+
def _reduce_29(val, _values, result)
|
365
|
+
result = define_atom(val.first)
|
366
|
+
result
|
367
|
+
end
|
368
|
+
|
369
|
+
# reduce 30 omitted
|
370
|
+
|
371
|
+
# reduce 31 omitted
|
372
|
+
|
373
|
+
# reduce 32 omitted
|
374
|
+
|
375
|
+
def _reduce_33(val, _values, result)
|
376
|
+
result = JSGF::Repetition.new(val[0], 0)
|
377
|
+
result
|
378
|
+
end
|
379
|
+
|
380
|
+
def _reduce_34(val, _values, result)
|
381
|
+
result = JSGF::Repetition.new(val[0], 1)
|
382
|
+
result
|
383
|
+
end
|
384
|
+
|
385
|
+
def _reduce_none(val, _values, result)
|
386
|
+
val[0]
|
387
|
+
end
|
388
|
+
|
389
|
+
end # class Parser
|
390
|
+
end # module JSGF
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'strscan'
|
2
|
+
|
3
|
+
module JSGF
|
4
|
+
class Tokenizer
|
5
|
+
TOKENS = {
|
6
|
+
/grammar/ => :GRAMMAR,
|
7
|
+
/#JSGF/ => :HEADER,
|
8
|
+
/import/ => :IMPORT,
|
9
|
+
/public/ => :PUBLIC,
|
10
|
+
/\{(\\.|[^\}]+)*\}/ => :TAG,
|
11
|
+
/[^ \t\n;=|*+<>\(\)\[\]{}*\/]+/ => :TOKEN,
|
12
|
+
%r{/\d*(\.\d+)?/} => :WEIGHT,
|
13
|
+
}
|
14
|
+
|
15
|
+
# @param io [IO] the {IO} stream to read from
|
16
|
+
def initialize(io)
|
17
|
+
io = io.read unless io.is_a?(String)
|
18
|
+
@scanner = StringScanner.new(io)
|
19
|
+
end
|
20
|
+
|
21
|
+
def next_token
|
22
|
+
return if @scanner.eos?
|
23
|
+
|
24
|
+
@scanner.skip(/\s+/) # Skip all leading whitespace
|
25
|
+
@scanner.skip(%r{//.*\n}) # Skip single-line comments
|
26
|
+
|
27
|
+
TOKENS.each do |(pattern, token)|
|
28
|
+
text = @scanner.scan(pattern)
|
29
|
+
return [token, text] if text
|
30
|
+
end
|
31
|
+
x = @scanner.getch
|
32
|
+
[x, x] unless x.nil?
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
data/lib/jsgf.rb
CHANGED
@@ -1,3 +1,19 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
require_relative 'jsgf/parser'
|
2
|
+
require_relative 'jsgf/tokenizer'
|
3
|
+
|
4
|
+
=begin
|
5
|
+
A parser for JSGF files.
|
6
|
+
|
7
|
+
http://www.w3.org/TR/jsgf/
|
8
|
+
|
9
|
+
# Usage
|
10
|
+
grammar = JSGF.read(filename)
|
11
|
+
=end
|
12
|
+
module JSGF
|
13
|
+
# @param filename [String] the file to parse
|
14
|
+
# @return [Grammar] the resulting {Grammar}
|
15
|
+
def self.read(filename)
|
16
|
+
tokenzier = JSGF::Tokenizer.new(File.read(filename))
|
17
|
+
JSGF::Parser.new(tokenzier).parse
|
18
|
+
end
|
3
19
|
end
|
data/test/jsgf/parser.rb
ADDED
@@ -0,0 +1,109 @@
|
|
1
|
+
require 'minitest/autorun'
|
2
|
+
require 'jsgf/parser'
|
3
|
+
|
4
|
+
describe JSGF::Parser do
|
5
|
+
describe 'when parsing the header' do
|
6
|
+
it 'must parse the version' do
|
7
|
+
grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;').parse
|
8
|
+
grammar.version.must_equal 'V1.0'
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'must parse the character encoding' do
|
12
|
+
grammar = JSGF::Parser.new('#JSGF V1.0 CHARSET; grammar header_grammar;').parse
|
13
|
+
grammar.version.must_equal 'V1.0'
|
14
|
+
grammar.character_encoding.must_equal 'CHARSET'
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'must parse the locale' do
|
18
|
+
grammar = JSGF::Parser.new('#JSGF V1.0 CHARSET locale; grammar header_grammar;').parse
|
19
|
+
grammar.version.must_equal 'V1.0'
|
20
|
+
grammar.character_encoding.must_equal 'CHARSET'
|
21
|
+
grammar.locale.must_equal 'locale'
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'must parse a rule with a single atom' do
|
26
|
+
grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one;').parse
|
27
|
+
grammar.rules.size.must_equal 1
|
28
|
+
grammar.rules.keys.must_equal ['rule']
|
29
|
+
grammar.rules['rule'][:atoms].size.must_equal 1
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'must parse a rule with multiple atoms' do
|
33
|
+
grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one two;').parse
|
34
|
+
grammar.rules.size.must_equal 1
|
35
|
+
grammar.rules.keys.must_equal ['rule']
|
36
|
+
grammar.rules['rule'][:atoms].size.must_equal 1
|
37
|
+
grammar.rules['rule'][:atoms].first.size.must_equal 2
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'must parse a rule with multiple grouped atoms' do
|
41
|
+
grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=(one two);').parse
|
42
|
+
grammar.rules.size.must_equal 1
|
43
|
+
grammar.rules.keys.must_equal ['rule']
|
44
|
+
grammar.rules['rule'][:atoms].size.must_equal 1
|
45
|
+
grammar.rules['rule'][:atoms].first.size.must_equal 2
|
46
|
+
end
|
47
|
+
|
48
|
+
describe 'when parsing an alternation' do
|
49
|
+
it 'must parse a rule with an alternation' do
|
50
|
+
grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one | two;').parse
|
51
|
+
grammar.rules.size.must_equal 1
|
52
|
+
grammar.rules.keys.must_equal ['rule']
|
53
|
+
grammar.rules['rule'][:atoms].size.must_equal 1
|
54
|
+
|
55
|
+
alternation = grammar.rules['rule'][:atoms].first
|
56
|
+
alternation.must_be_kind_of JSGF::Alternation
|
57
|
+
alternation.size.must_equal 2
|
58
|
+
end
|
59
|
+
|
60
|
+
it 'must parse a rule with a bigger alternation' do
|
61
|
+
grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one | two | three;').parse
|
62
|
+
grammar.rules.size.must_equal 1
|
63
|
+
grammar.rules.keys.must_equal ['rule']
|
64
|
+
grammar.rules['rule'][:atoms].size.must_equal 1
|
65
|
+
|
66
|
+
alternation = grammar.rules['rule'][:atoms].first
|
67
|
+
alternation.must_be_kind_of JSGF::Alternation
|
68
|
+
alternation.size.must_equal 3
|
69
|
+
alternation.elements[0][:atom].must_equal 'one'
|
70
|
+
alternation.elements[1][:atom].must_equal 'two'
|
71
|
+
alternation.elements[2][:atom].must_equal 'three'
|
72
|
+
end
|
73
|
+
|
74
|
+
it 'must parse a weighted alternation' do
|
75
|
+
grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>= /0.5/ one | /0.25/ two | /0.25/ three;').parse
|
76
|
+
grammar.rules.keys.must_equal ['rule']
|
77
|
+
grammar.rules['rule'][:atoms].size.must_equal 1
|
78
|
+
|
79
|
+
alternation = grammar.rules['rule'][:atoms].first
|
80
|
+
alternation.must_be_kind_of JSGF::Alternation
|
81
|
+
alternation.size.must_equal 3
|
82
|
+
alternation.weights.must_equal [0.5, 0.25, 0.25]
|
83
|
+
alternation.elements[0][:atom].must_equal 'one'
|
84
|
+
end
|
85
|
+
|
86
|
+
it 'must parse a grouped alternation' do
|
87
|
+
grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=(one | two | three);').parse
|
88
|
+
grammar.rules.size.must_equal 1
|
89
|
+
grammar.rules.keys.must_equal ['rule']
|
90
|
+
grammar.rules['rule'][:atoms].size.must_equal 1
|
91
|
+
grammar.rules['rule'][:atoms].first.must_be_kind_of JSGF::Alternation
|
92
|
+
grammar.rules['rule'][:atoms].first.size.must_equal 3
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
it 'must parse an optional group of a single atom' do
|
97
|
+
grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=[one];').parse
|
98
|
+
grammar.rules.size.must_equal 1
|
99
|
+
grammar.rules.keys.must_equal ['rule']
|
100
|
+
grammar.rules['rule'][:atoms].size.must_equal 1
|
101
|
+
end
|
102
|
+
|
103
|
+
it 'must parse an optional alternation' do
|
104
|
+
grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=[one | two];').parse
|
105
|
+
grammar.rules.size.must_equal 1
|
106
|
+
grammar.rules.keys.must_equal ['rule']
|
107
|
+
grammar.rules['rule'][:atoms].size.must_equal 1
|
108
|
+
end
|
109
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'minitest/autorun'
|
2
|
+
require 'jsgf/tokenizer'
|
3
|
+
|
4
|
+
describe JSGF::Tokenizer do
|
5
|
+
it 'must pass individual characters' do
|
6
|
+
JSGF::Tokenizer.new(';').next_token.must_equal [';', ';']
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'must recognize the grammar keyword' do
|
10
|
+
JSGF::Tokenizer.new('grammar').next_token.must_equal [:GRAMMAR, 'grammar']
|
11
|
+
end
|
12
|
+
|
13
|
+
it 'must skip leading whitespace' do
|
14
|
+
JSGF::Tokenizer.new(' grammar').next_token.must_equal [:GRAMMAR, 'grammar']
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'must recognize the header string' do
|
18
|
+
JSGF::Tokenizer.new('#JSGF').next_token.must_equal [:HEADER, '#JSGF']
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'must recognize the public keyword' do
|
22
|
+
JSGF::Tokenizer.new('public').next_token.must_equal [:PUBLIC, 'public']
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'must recognize a tag block ' do
|
26
|
+
JSGF::Tokenizer.new('{ some stuff }').next_token.must_equal [:TAG, '{ some stuff }']
|
27
|
+
end
|
28
|
+
|
29
|
+
it 'must recognize a weight' do
|
30
|
+
JSGF::Tokenizer.new('/0.5/').next_token.must_equal [:WEIGHT, '/0.5/']
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'must recognize a token' do
|
34
|
+
JSGF::Tokenizer.new(' some_token ').next_token.must_equal [:TOKEN, 'some_token']
|
35
|
+
end
|
36
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jsgf
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '0'
|
4
|
+
version: '0.1'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brandon Fosdick
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-01-
|
11
|
+
date: 2015-01-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -51,6 +51,14 @@ files:
|
|
51
51
|
- Rakefile
|
52
52
|
- jsgf.gemspec
|
53
53
|
- lib/jsgf.rb
|
54
|
+
- lib/jsgf/alternation.rb
|
55
|
+
- lib/jsgf/grammar.rb
|
56
|
+
- lib/jsgf/optional.rb
|
57
|
+
- lib/jsgf/parser.rb
|
58
|
+
- lib/jsgf/repetition.rb
|
59
|
+
- lib/jsgf/tokenizer.rb
|
60
|
+
- test/jsgf/parser.rb
|
61
|
+
- test/jsgf/tokenizer.rb
|
54
62
|
homepage: http://github.com/bfoz/jsgf-ruby
|
55
63
|
licenses:
|
56
64
|
- BSD
|
@@ -75,4 +83,6 @@ rubygems_version: 2.2.2
|
|
75
83
|
signing_key:
|
76
84
|
specification_version: 4
|
77
85
|
summary: Java Speech Grammar Format
|
78
|
-
test_files:
|
86
|
+
test_files:
|
87
|
+
- test/jsgf/parser.rb
|
88
|
+
- test/jsgf/tokenizer.rb
|