jsgf 0.1 → 0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 66ac3ad9f73f24c8c44884964606916ad2495680
-  data.tar.gz: 1a67637a02d934eee7ee0c53f8586217f68a6567
+  metadata.gz: fdf13c6642d0aa9a9faaa005d72563af8151ac68
+  data.tar.gz: 3014db9c26234c9744262e9c50a0198c9c81b466
 SHA512:
-  metadata.gz: 6b42838dcefd7eb536f0dd61a9c6ca352fba12b60577960ef316c35dda5d9ce0e681a34ba8b2dfcd70d11b6981f6c22a70700a6cdf806dfd538671df35789a73
-  data.tar.gz: c04356733209a868672753df49626aa595f9b56c91e2a3dfd95493c75599df244ac3b4d93c09f94a54f2e582c51895f02396ec120605e9234830941868534953
+  metadata.gz: 140f2abe3f92727e630bf4aab4e4f33ba92afa09fc57e38f2e50d6669769a60fdc23ef0aa6c2565fa68b925874bffbec645507f6fd4afdc10500c3272293e907
+  data.tar.gz: aae7b624c87f88c923a2d6db6bb8e63ea6963860d50178cfb0b5128a74b4aee223abef8754343db1e725af6bff5ba93a6c124bbae0b08a0cad070d0069154825

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+- 2.0.0
+- 2.1.1

data/README.md

@@ -1,8 +1,22 @@
-# Jsgf
+Jsgf
+====
 
+[![Build Status](https://travis-ci.org/bfoz/jsgf-ruby.png)](https://travis-ci.org/bfoz/jsgf-ruby)
 [![Gem Version](https://badge.fury.io/rb/jsgf.svg)](http://badge.fury.io/rb/jsgf)
 
-## Installation
+For all of your [Java Speech Grammar Format](http://www.w3.org/TR/jsgf/) parsing needs.
+
+Usage
+-----
+
+```ruby
+require 'jsgf'
+
+grammar = JSGF.read('example.gram')
+```
+
+Installation
+------------
 
 Add this line to your application's Gemfile:
 
@@ -18,14 +32,7 @@ Or install it yourself as:
 
     $ gem install jsgf
 
-## Usage
-
-TODO: Write usage instructions here
-
-## Contributing
+License
+-------
 
-1. Fork it ( https://github.com/[my-github-username]/jsgf/fork )
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create a new Pull Request
+Copyright 2015 Brandon Fosdick <bfoz@bfoz.net> and released under the BSD license.

data/jsgf.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
     spec.name          = "jsgf"
-    spec.version       = '0.1'
+    spec.version       = '0.2'
     spec.authors       = ["Brandon Fosdick"]
     spec.email         = ["bfoz@bfoz.net"]
     spec.summary       = %q{Java Speech Grammar Format}

data/lib/jsgf/alternation.rb

@@ -1,7 +1,7 @@
 module JSGF
     class Alternation
 	attr_reader :elements
-	attr_writer :optional
+	attr_accessor :optional
 	attr_reader :tags
 
 	def initialize(*args)

data/lib/jsgf/grammar.rb

@@ -2,13 +2,15 @@ module JSGF
     class Grammar
 	attr_reader :character_encoding
 	attr_reader :locale
+	attr_reader :grammar_name
 	attr_reader :private_rules
 	attr_reader :public_rules
 	attr_reader :version
 
-	def initialize(character_encoding:nil, locale:nil, private_rules:{}, public_rules:{}, version:nil)
+	def initialize(name:nil, character_encoding:nil, locale:nil, private_rules:{}, public_rules:{}, version:nil)
 	    @character_encoding = character_encoding
 	    @locale = locale
+	    @grammar_name = name
 	    @private_rules = private_rules
 	    @public_rules = public_rules
 	    @version = version
@@ -19,5 +21,48 @@ module JSGF
 	def rules
 	    @public_rules.merge(@private_rules)
 	end
+
+	def to_s
+	    private_rule_array = @private_rules.map do |(name, rule)|
+		atoms = rule.map {|a| unparse_atom(a) }.join(' ')
+		"<#{name}> = #{atoms};"
+	    end
+	    public_rule_array = @public_rules.map do |(name, rule)|
+		atoms = rule.map {|a| unparse_atom(a) }.join(' ')
+		"public <#{name}> = #{atoms};"
+	    end
+
+	    [header, grammar_header, *public_rule_array, *private_rule_array].join('\n')
+	end
+
+    private
+	# Generate the grammar name header line
+	def grammar_header
+	    "grammar #{grammar_name};"
+	end
+
+	# Generate the JSGF header line
+	def header
+	    ['#JSGF', version, character_encoding, locale].compact.join(' ') + ';'
+	end
+
+	def unparse_atom(atom, nested:false)
+	    case atom
+		when Array
+		    s = atom.map {|a| unparse_atom(a, nested:nested)}.join(' ')
+		    nested ? ('(' + s + ')') : s
+		when Alternation
+		    s = atom.elements.map {|a| unparse_atom(a, nested:true)}.join(' | ')
+		    atom.optional ? ('[' + s + ']') : s
+		when Optional then '[' + atom.elements.map {|a| unparse_atom(a, nested:nested)}.join(' | ') + ']'
+		else
+		    weight = (atom[:weight] != 1.0) ? "/#{atom[:weight]}/" : nil
+		    if atom[:name]
+			[weight, '<' + atom[:name] + '>'].compact.join(' ')
+		    else
+			[weight, atom[:atom], *atom[:tags]].compact.join(' ')
+		    end
+	    end
+	end
     end
 end

data/lib/jsgf/optional.rb

@@ -1,5 +1,7 @@
 module JSGF
     class Optional
+	attr_reader :elements
+
 	def initialize(*args)
 	    @elements = args
 	end

data/lib/jsgf/parser.rb

@@ -11,6 +11,7 @@ require_relative 'alternation'
 require_relative 'grammar'
 require_relative 'optional'
 require_relative 'repetition'
+require_relative 'tokenizer'
 
 module JSGF
   class Parser < Racc::Parser
@@ -22,6 +23,7 @@ attr_reader :handler
 def initialize(tokenizer)
     @private_rules = {}
     @public_rules = {}
+    @rules = {}
     tokenizer = JSGF::Tokenizer.new(tokenizer) if tokenizer.is_a?(String)
     @tokenizer = tokenizer
     super()
@@ -50,22 +52,32 @@ def define_atom(atom, weight=1.0, tags=[])
 end
 
 def define_rule(name, visibility=:private, *args)
-    r = {name:name, visibility:visibility, atoms:args}
-    if visibility == :private
-	@private_rules[name] = r
-    else
-	@public_rules[name] = r
-    end
+    r = {name: name, visibility:visibility, atoms:args.flatten}
+    @rules[name] = r
     r
 end
 
+def rule_reference(name)
+    {name:name, weight:1.0, tags:[]}
+end
+
 def next_token
     @tokenizer.next_token
-end
+  end
 
 def parse
     do_parse
-    JSGF::Grammar.new(	character_encoding:@charset,
+
+    @rules.each do |(k,v)|
+	if v[:visibility] == :private
+	    @private_rules[k] = v[:atoms]
+	else
+	    @public_rules[k] = v[:atoms]
+	end
+    end
+
+    JSGF::Grammar.new(	name:@grammar_name,
+			character_encoding:@charset,
 			locale:@locale,
 			private_rules:@private_rules,
 			public_rules:@public_rules,
@@ -74,62 +86,62 @@ end
 ##### State transition tables begin ###
 
 racc_action_table = [
-    40,    37,    17,    11,    16,    40,    37,    38,    11,    39,
-    40,    37,    38,    11,    39,    40,    44,    38,    11,    39,
-    40,    37,    38,    11,    39,    40,    37,    38,    11,    39,
-    40,    37,    38,    11,    39,    40,    37,    38,    11,    39,
-     9,    13,    38,    45,    39,    13,    11,     9,    13,    13,
-    11,    49,    48,    11,    11,    61,    49,    62,    49,    51,
-    60,    49,    28,    47,    27,    46,    52,    53,    52,    53,
-    30,    29,    26,    11,    24,    58,    23,    51,    11,    18,
-    15,     5,     4 ]
+   -21,     9,    13,    28,   -21,    27,   -21,    11,   -21,    52,
+    53,    40,    37,    17,    11,    16,    40,    37,    38,    11,
+    39,    40,    13,    38,    11,    39,    44,    11,    38,    40,
+    39,    49,    11,    30,    40,    37,    38,    11,    39,    40,
+    29,    38,    11,    39,    40,    37,    38,    11,    39,     9,
+    13,    38,    13,    39,    13,    11,    47,    11,    54,    11,
+    48,    46,    26,    45,    52,    53,    52,    53,    52,    53,
+    24,    23,    60,    11,    18,    54,    15,    48,     5,    63,
+    64,     4 ]
 
 racc_action_check = [
-    59,    59,     4,    59,     4,    32,    32,    59,    32,    59,
-    24,    24,    32,    24,    32,    37,    25,    24,    37,    24,
-    38,    38,    37,    38,    37,    39,    39,    38,    39,    38,
-    49,    49,    39,    49,    39,    44,    44,    49,    44,    49,
-     2,     2,    44,    26,    44,     6,     2,     7,     7,    20,
-     6,    56,    31,     7,    20,    56,    31,    57,    55,    33,
-    55,    57,    17,    28,    17,    28,    35,    35,    54,    54,
-    23,    22,    15,    13,    12,    47,    11,    50,     9,     5,
-     3,     1,     0 ]
+    34,     7,     7,    17,    34,    17,    34,     7,    34,    34,
+    34,    48,    48,     4,    48,     4,    38,    38,    48,    38,
+    48,    37,    20,    38,    37,    38,    26,    20,    37,    32,
+    37,    32,    32,    23,    24,    24,    32,    24,    32,    57,
+    22,    24,    57,    24,    39,    39,    57,    39,    57,     2,
+     2,    39,    13,    39,     6,     2,    31,    13,    36,     6,
+    31,    28,    15,    28,    50,    50,    55,    55,    62,    62,
+    12,    11,    46,     9,     5,    51,     3,    56,     1,    58,
+    59,     0 ]
 
 racc_action_pointer = [
-    79,    81,    36,    78,    -5,    79,    40,    43,   nil,    68,
-   nil,    69,    62,    63,   nil,    65,   nil,    55,   nil,   nil,
-    44,   nil,    62,    59,     3,     4,    34,   nil,    56,   nil,
-   nil,    43,    -2,    53,   nil,    48,   nil,     8,    13,    18,
-   nil,   nil,   nil,   nil,    28,   nil,   nil,    66,   nil,    23,
-    71,   nil,   nil,   nil,    50,    45,    38,    48,   nil,    -7,
-   nil,   nil,   nil ]
+    78,    78,    45,    74,     6,    74,    49,    -3,   nil,    63,
+   nil,    64,    58,    47,   nil,    55,   nil,    -4,   nil,   nil,
+    17,   nil,    31,    22,    27,   nil,    17,   nil,    54,   nil,
+   nil,    47,    22,   nil,    -9,   nil,    52,    14,     9,    37,
+   nil,   nil,   nil,   nil,   nil,   nil,    63,   nil,     4,   nil,
+    46,    69,   nil,   nil,   nil,    48,    64,    32,    64,    63,
+   nil,   nil,    50,   nil,   nil ]
 
 racc_action_default = [
-   -35,   -35,    -1,   -35,   -35,   -35,    -2,   -35,   -10,   -35,
-   -13,   -35,   -35,   -35,    -4,   -35,    -5,   -35,    63,   -14,
-    -3,   -11,   -35,   -35,   -35,   -35,   -35,    -6,   -35,   -12,
-   -15,   -35,   -18,   -20,   -22,   -24,   -25,   -35,   -35,   -35,
-   -29,   -30,   -31,   -32,   -35,    -9,    -7,   -35,   -16,   -35,
-   -21,   -23,   -33,   -34,   -26,   -35,   -35,   -35,    -8,   -19,
-   -27,   -28,   -17 ]
+   -38,   -38,    -1,   -38,   -38,   -38,    -2,   -38,   -10,   -38,
+   -13,   -38,   -38,   -38,    -4,   -38,    -5,   -38,    65,   -14,
+    -3,   -11,   -38,   -38,   -38,   -18,   -38,    -6,   -38,   -12,
+   -15,   -38,   -38,   -19,   -23,   -22,   -25,   -38,   -38,   -38,
+   -32,   -33,   -34,   -35,    -9,    -7,   -38,   -16,   -38,   -17,
+   -23,   -26,   -36,   -37,   -24,   -27,   -28,   -29,   -38,   -38,
+    -8,   -20,   -21,   -30,   -31 ]
 
 racc_goto_table = [
-    50,    31,    12,    19,     1,     6,    12,    12,    14,    22,
-    20,     3,     7,    25,     8,    55,    56,    19,    59,    21,
-    12,    57,     2,    54,   nil,   nil,   nil,    50 ]
+    50,    51,    12,    19,    14,    55,    12,    12,     3,    22,
+    25,    58,    59,    12,     6,    31,    62,    19,     8,    20,
+    12,    32,    61,    21,     7,    50,    51,     2,     1 ]
 
 racc_goto_check = [
-    12,    10,     8,     9,     1,     3,     8,     8,     6,     8,
-     3,     5,     4,     8,     7,    10,    10,     9,    11,     7,
-     8,    10,     2,    14,   nil,   nil,   nil,    12 ]
+    13,    15,     8,     9,     6,    13,     8,     8,     5,     8,
+     9,    16,    16,     8,     3,    10,    13,     9,     7,     3,
+     8,    11,    12,     7,     4,    13,    15,     2,     1 ]
 
 racc_goto_pointer = [
-   nil,     4,    22,     3,    10,    11,     5,    12,     0,    -3,
-   -23,   -31,   -32,   nil,   -14,   nil,   nil,   nil ]
+   nil,    28,    27,    12,    22,     8,     1,    16,     0,    -3,
+    -9,    -3,   -26,   -32,   nil,   -31,   -27,   nil,   nil ]
 
 racc_goto_default = [
    nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,    41,    10,
-   nil,    32,    33,    34,    35,    36,    42,    43 ]
+    56,    57,    33,    34,    35,    36,   nil,    42,    43 ]
 
 racc_reduce_table = [
   0, 0, :racc_error,
@@ -149,28 +161,31 @@ racc_reduce_table = [
   2, 23, :_reduce_none,
   3, 28, :_reduce_15,
   4, 29, :_reduce_16,
-  5, 29, :_reduce_17,
-  1, 30, :_reduce_18,
-  3, 30, :_reduce_19,
-  1, 31, :_reduce_20,
-  2, 31, :_reduce_21,
+  4, 29, :_reduce_17,
+  2, 29, :_reduce_18,
+  1, 30, :_reduce_19,
+  3, 30, :_reduce_20,
   1, 32, :_reduce_none,
-  2, 32, :_reduce_23,
+  1, 32, :_reduce_none,
+  1, 35, :_reduce_none,
+  2, 35, :_reduce_24,
+  1, 31, :_reduce_none,
+  2, 31, :_reduce_26,
+  2, 34, :_reduce_27,
+  1, 36, :_reduce_none,
+  1, 36, :_reduce_none,
+  3, 37, :_reduce_30,
+  3, 38, :_reduce_31,
+  1, 33, :_reduce_32,
+  1, 33, :_reduce_33,
   1, 33, :_reduce_none,
   1, 33, :_reduce_none,
-  2, 35, :_reduce_26,
-  3, 36, :_reduce_27,
-  3, 37, :_reduce_28,
-  1, 34, :_reduce_29,
-  1, 34, :_reduce_none,
-  1, 34, :_reduce_none,
-  1, 34, :_reduce_none,
-  2, 34, :_reduce_33,
-  2, 34, :_reduce_34 ]
+  2, 33, :_reduce_36,
+  2, 33, :_reduce_37 ]
 
-racc_reduce_n = 35
+racc_reduce_n = 38
 
-racc_shift_n = 63
+racc_shift_n = 65
 
 racc_token_table = {
   false => 0,
@@ -245,12 +260,13 @@ Racc_token_to_s_table = [
   "import_statement",
   "rule_name",
   "rule",
-  "alternate_list",
-  "rule_expansion",
-  "tagged_rule_item",
-  "rule_item",
+  "alternation",
+  "atom_list",
+  "alternation_item",
   "rule_atom",
   "weighted_item",
+  "tagged_atom",
+  "group_list",
   "rule_group",
   "rule_optional" ]
 
@@ -306,78 +322,87 @@ def _reduce_15(val, _values, result)
 end
 
 def _reduce_16(val, _values, result)
- define_rule(val[0], :private, *(val[2..-2])) 
+ result = define_rule(val[0], :private, *(val[2..-2])) 
     result
 end
 
 def _reduce_17(val, _values, result)
- define_rule(val[1], :public, val[3]) 
+ result = define_rule(val[0], :private, *(val[2..-2])) 
     result
 end
 
 def _reduce_18(val, _values, result)
- result = val.first 
+ val[1][:visibility] = :public 
     result
 end
 
 def _reduce_19(val, _values, result)
- result = define_alternation(*val) 
+ result = val.first 
     result
 end
 
 def _reduce_20(val, _values, result)
- result = val.first 
+ result = define_alternation(*val) 
     result
 end
 
-def _reduce_21(val, _values, result)
- result = val 
-    result
-end
+# reduce 21 omitted
 
 # reduce 22 omitted
 
-def _reduce_23(val, _values, result)
+# reduce 23 omitted
+
+def _reduce_24(val, _values, result)
  val[0][:tags].push(val[1]); result = val[0] 
     result
 end
 
-# reduce 24 omitted
-
 # reduce 25 omitted
 
 def _reduce_26(val, _values, result)
- val[1][:weight] = val.first[1..-2].to_f; result = val[1] 
+ result = val; 
     result
 end
 
 def _reduce_27(val, _values, result)
+ val[1][:weight] = val.first[1..-2].to_f; result = val[1] 
+    result
+end
+
+# reduce 28 omitted
+
+# reduce 29 omitted
+
+def _reduce_30(val, _values, result)
  result = val[1] 
     result
 end
 
-def _reduce_28(val, _values, result)
+def _reduce_31(val, _values, result)
  result = define_optional(val[1]) 
     result
 end
 
-def _reduce_29(val, _values, result)
+def _reduce_32(val, _values, result)
  result = define_atom(val.first) 
     result
 end
 
-# reduce 30 omitted
+def _reduce_33(val, _values, result)
+ result = rule_reference(val[0]) 
+    result
+end
 
-# reduce 31 omitted
+# reduce 34 omitted
 
-# reduce 32 omitted
+# reduce 35 omitted
 
-def _reduce_33(val, _values, result)
+def _reduce_36(val, _values, result)
  result = JSGF::Repetition.new(val[0], 0) 
     result
 end
 
-def _reduce_34(val, _values, result)
+def _reduce_37(val, _values, result)
  result = JSGF::Repetition.new(val[0], 1) 
     result
 end

data/lib/jsgf/tokenizer.rb

@@ -24,6 +24,12 @@ module JSGF
 	    @scanner.skip(/\s+/)	# Skip all leading whitespace
 	    @scanner.skip(%r{//.*\n})	# Skip single-line comments
 
+	    # Skip C-style comments
+	    if @scanner.scan(%r{/\*})
+		# Look for the end of the block, and skip any trailing whitespace
+		@scanner.skip_until(%r{\*/\s*})
+	    end
+
 	    TOKENS.each do |(pattern, token)|
 		text = @scanner.scan(pattern)
 		return [token, text] if text

data/test/jsgf/grammar.rb

@@ -0,0 +1,66 @@
+require 'minitest/autorun'
+require 'jsgf/parser'
+
+describe JSGF::Grammar do
+    it 'must unparse a header' do
+	grammar = JSGF::Parser.new('#JSGF; grammar header_grammar;').parse
+	grammar.to_s.must_equal '#JSGF;\ngrammar header_grammar;'
+    end
+
+    it 'must unparse a header with a version' do
+	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;').parse
+	grammar.to_s.must_equal '#JSGF V1.0;\ngrammar header_grammar;'
+    end
+
+    it 'must unparse a header with a version and a character encoding' do
+	grammar = JSGF::Parser.new('#JSGF V1.0 ENCODING; grammar header_grammar;').parse
+	grammar.to_s.must_equal '#JSGF V1.0 ENCODING;\ngrammar header_grammar;'
+    end
+
+    it 'must unparse a simple rule' do
+	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one;').parse
+	grammar.to_s.must_equal '#JSGF V1.0;\ngrammar header_grammar;\n<rule> = one;'
+    end
+
+    it 'must unparse a public rule' do
+	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;public <rule>=one;').parse
+	grammar.to_s.must_equal '#JSGF V1.0;\ngrammar header_grammar;\npublic <rule> = one;'
+    end
+
+    it 'must unparse a multi-atom rule' do
+	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one two;').parse
+	grammar.to_s.must_equal '#JSGF V1.0;\ngrammar header_grammar;\n<rule> = one two;'
+    end
+
+    it 'must unparse a multi-atom optional' do
+	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=[one two];').parse
+	grammar.to_s.must_equal '#JSGF V1.0;\ngrammar header_grammar;\n<rule> = [one two];'
+    end
+
+    it 'must unparse an alternation' do
+	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one | two;').parse
+	grammar.to_s.must_equal '#JSGF V1.0;\ngrammar header_grammar;\n<rule> = one | two;'
+    end
+
+    it 'must unparse an optional alternation' do
+	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=[one | two];').parse
+	grammar.to_s.must_equal '#JSGF V1.0;\ngrammar header_grammar;\n<rule> = [one | two];'
+    end
+
+    it 'must unparse an alternation with a nested group' do
+	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=(one two) | three;').parse
+	grammar.to_s.must_equal '#JSGF V1.0;\ngrammar header_grammar;\n<rule> = (one two) | three;'
+    end
+
+    describe 'when unparsing rule references' do
+	it 'must unparse a rule reference' do
+	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=(one <rule2>) | three; <rule2>=two;').parse
+	    grammar.to_s.must_equal '#JSGF V1.0;\ngrammar header_grammar;\n<rule> = (one <rule2>) | three;\n<rule2> = two;'
+	end
+
+	it 'must unparse a weighted rule reference' do
+	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>= /0.5/ one | /0.5/ <rule2>; <rule2>=two;').parse
+	    grammar.to_s.must_equal '#JSGF V1.0;\ngrammar header_grammar;\n<rule> = /0.5/ one | /0.5/ <rule2>;\n<rule2> = two;'
+	end
+    end
+end

data/test/jsgf/parser.rb

@@ -26,23 +26,21 @@ describe JSGF::Parser do
 	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one;').parse
 	grammar.rules.size.must_equal 1
 	grammar.rules.keys.must_equal ['rule']
-	grammar.rules['rule'][:atoms].size.must_equal 1
+	grammar.rules['rule'].size.must_equal 1
     end
 
     it 'must parse a rule with multiple atoms' do
 	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one two;').parse
 	grammar.rules.size.must_equal 1
 	grammar.rules.keys.must_equal ['rule']
-	grammar.rules['rule'][:atoms].size.must_equal 1
-	grammar.rules['rule'][:atoms].first.size.must_equal 2
+	grammar.rules['rule'].size.must_equal 2
     end
 
     it 'must parse a rule with multiple grouped atoms' do
 	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=(one two);').parse
 	grammar.rules.size.must_equal 1
 	grammar.rules.keys.must_equal ['rule']
-	grammar.rules['rule'][:atoms].size.must_equal 1
-	grammar.rules['rule'][:atoms].first.size.must_equal 2
+	grammar.rules['rule'].size.must_equal 2
     end
 
     describe 'when parsing an alternation' do
@@ -50,60 +48,98 @@ describe JSGF::Parser do
 	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one | two;').parse
 	    grammar.rules.size.must_equal 1
 	    grammar.rules.keys.must_equal ['rule']
-	    grammar.rules['rule'][:atoms].size.must_equal 1
+	    grammar.rules['rule'].size.must_equal 1
 
-	    alternation = grammar.rules['rule'][:atoms].first
+	    alternation = grammar.rules['rule'].first
 	    alternation.must_be_kind_of JSGF::Alternation
 	    alternation.size.must_equal 2
 	end
-	
+
 	it 'must parse a rule with a bigger alternation' do
 	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one | two | three;').parse
 	    grammar.rules.size.must_equal 1
 	    grammar.rules.keys.must_equal ['rule']
-	    grammar.rules['rule'][:atoms].size.must_equal 1
+	    grammar.rules['rule'].size.must_equal 1
 
-	    alternation = grammar.rules['rule'][:atoms].first
+	    alternation = grammar.rules['rule'].first
 	    alternation.must_be_kind_of JSGF::Alternation
 	    alternation.size.must_equal 3
-	    alternation.elements[0][:atom].must_equal 'one'
-	    alternation.elements[1][:atom].must_equal 'two'
-	    alternation.elements[2][:atom].must_equal 'three'
+	    alternation.elements[0].must_equal Hash[atom:'one', weight:1.0, tags:[]]
+	    alternation.elements[1].must_equal Hash[atom:'two', weight:1.0, tags:[]]
+	    alternation.elements[2].must_equal Hash[atom:'three', weight:1.0, tags:[]]
 	end
 
 	it 'must parse a weighted alternation' do
 	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>= /0.5/ one | /0.25/ two | /0.25/ three;').parse
 	    grammar.rules.keys.must_equal ['rule']
-	    grammar.rules['rule'][:atoms].size.must_equal 1
+	    grammar.rules['rule'].size.must_equal 1
 
-	    alternation = grammar.rules['rule'][:atoms].first
+	    alternation = grammar.rules['rule'].first
 	    alternation.must_be_kind_of JSGF::Alternation
 	    alternation.size.must_equal 3
 	    alternation.weights.must_equal [0.5, 0.25, 0.25]
-	    alternation.elements[0][:atom].must_equal 'one'
+	    alternation.elements[0].must_equal Hash[atom:'one', weight:0.5, tags:[]]
 	end
 
 	it 'must parse a grouped alternation' do
 	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=(one | two | three);').parse
 	    grammar.rules.size.must_equal 1
 	    grammar.rules.keys.must_equal ['rule']
-	    grammar.rules['rule'][:atoms].size.must_equal 1
-	    grammar.rules['rule'][:atoms].first.must_be_kind_of JSGF::Alternation
-	    grammar.rules['rule'][:atoms].first.size.must_equal 3
+	    grammar.rules['rule'].size.must_equal 1
+	    grammar.rules['rule'].first.must_be_kind_of JSGF::Alternation
+	    grammar.rules['rule'].first.size.must_equal 3
 	end
     end
 
-    it 'must parse an optional group of a single atom' do
-	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=[one];').parse
-	grammar.rules.size.must_equal 1
-	grammar.rules.keys.must_equal ['rule']
-	grammar.rules['rule'][:atoms].size.must_equal 1
+    describe 'when parsing optionals' do
+	it 'must parse an optional group of a single atom' do
+	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=[one];').parse
+	    grammar.rules.size.must_equal 1
+	    grammar.rules.keys.must_equal ['rule']
+	    grammar.rules['rule'].size.must_equal 1
+	end
+
+	it 'must parse an optional alternation' do
+	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=[one | two];').parse
+	    grammar.rules.size.must_equal 1
+	    grammar.rules.keys.must_equal ['rule']
+	    grammar.rules['rule'].size.must_equal 1
+
+	    rule = grammar.rules['rule']
+	    rule.first.must_be_kind_of JSGF::Alternation
+	    rule.first.optional.must_equal true
+	end
+
+	it 'must parse a multi-atom optional alternation' do
+	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one two [three | four];').parse
+	    grammar.rules.size.must_equal 1
+	    grammar.rules.keys.must_equal ['rule']
+	    grammar.rules['rule'].size.must_equal 3
+
+	    rule = grammar.rules['rule']
+	    rule.size.must_equal 3
+	end
     end
 
-    it 'must parse an optional alternation' do
-	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=[one | two];').parse
-	grammar.rules.size.must_equal 1
-	grammar.rules.keys.must_equal ['rule']
-	grammar.rules['rule'][:atoms].size.must_equal 1
+    describe 'when parsing rule references' do
+	it 'must parse a rule reference' do
+	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=<rule1>;<rule1>=one;').parse
+	    grammar.rules.size.must_equal 2
+	    grammar.rules.keys.must_equal ['rule', 'rule1']
+	    grammar.rules['rule'].must_equal [{name:'rule1', weight:1.0, tags:[]}]
+	    grammar.rules['rule1'].must_equal [{atom:'one', weight:1.0, tags:[]}]
+	end
+
+	it 'must parse a weighted rule reference' do
+	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>= /0.5/ <rule1> | /0.5/ two; <rule1>=one;').parse
+	    grammar.rules.size.must_equal 2
+	    grammar.rules.keys.must_equal ['rule', 'rule1']
+
+	    grammar.rules['rule'].size.must_equal 1
+	    grammar.rules['rule'].first.must_be_kind_of JSGF::Alternation
+	    grammar.rules['rule'].first.elements.must_equal [{name:'rule1', weight:0.5, tags:[]}, {atom:'two', weight:0.5, tags:[]}]
+
+	    grammar.rules['rule1'].must_equal [{atom:'one', weight:1.0, tags:[]}]
+	end
     end
 end

data/test/jsgf/tokenizer.rb

@@ -33,4 +33,10 @@ describe JSGF::Tokenizer do
     it 'must recognize a token' do
 	JSGF::Tokenizer.new(' some_token ').next_token.must_equal [:TOKEN, 'some_token']
     end
+
+    describe 'when given comments' do
+	it 'must skip C-style comments' do
+	    JSGF::Tokenizer.new(' /* a comment */ some_token ').next_token.must_equal [:TOKEN, 'some_token']
+	end
+    end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jsgf
 version: !ruby/object:Gem::Version
-  version: '0.1'
+  version: '0.2'
 platform: ruby
 authors:
 - Brandon Fosdick
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-04 00:00:00.000000000 Z
+date: 2015-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - README.md
 - Rakefile
@@ -57,6 +58,7 @@ files:
 - lib/jsgf/parser.rb
 - lib/jsgf/repetition.rb
 - lib/jsgf/tokenizer.rb
+- test/jsgf/grammar.rb
 - test/jsgf/parser.rb
 - test/jsgf/tokenizer.rb
 homepage: http://github.com/bfoz/jsgf-ruby
@@ -84,5 +86,6 @@ signing_key:
 specification_version: 4
 summary: Java Speech Grammar Format
 test_files:
+- test/jsgf/grammar.rb
 - test/jsgf/parser.rb
 - test/jsgf/tokenizer.rb