jsgf 0 → 0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 99ebe95b6502d991b3aa978503305a4c6f2dca47
-  data.tar.gz: b15f9944f56e9a289694836b0b84a6c9733a1770
+  metadata.gz: 66ac3ad9f73f24c8c44884964606916ad2495680
+  data.tar.gz: 1a67637a02d934eee7ee0c53f8586217f68a6567
 SHA512:
-  metadata.gz: 129c77802146376e42ab2c72197807c1c0dc126a0561c542856c0cb9c6d5103fe749263ba7b5f168c3c95e18d68476287c59dabb1ef5883952f47fd0421ab032
-  data.tar.gz: 3d312bf34c5b5041dda707f834194c669d5c467d4e6daea5b80fce2e73860fbec9d23463b8e689062ecf5ae68b3f61b2b5108a0ed922b632c924a724638e357a
+  metadata.gz: 6b42838dcefd7eb536f0dd61a9c6ca352fba12b60577960ef316c35dda5d9ce0e681a34ba8b2dfcd70d11b6981f6c22a70700a6cdf806dfd538671df35789a73
+  data.tar.gz: c04356733209a868672753df49626aa595f9b56c91e2a3dfd95493c75599df244ac3b4d93c09f94a54f2e582c51895f02396ec120605e9234830941868534953

data/Gemfile

@@ -2,3 +2,7 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in jsgf.gemspec
 gemspec
+
+group :development do
+    gem 'racc'
+end

data/README.md

@@ -1,6 +1,6 @@
 # Jsgf
 
-TODO: Write a gem description
+[![Gem Version](https://badge.fury.io/rb/jsgf.svg)](http://badge.fury.io/rb/jsgf)
 
 ## Installation
 

data/Rakefile

@@ -1,2 +1,18 @@
-require "bundler/gem_tasks"
+require 'bundler/gem_tasks'
+require 'rake/testtask'
 
+rule '.rb' => '.y' do |t|
+    `racc -l -o #{t.name} #{t.source}`
+end
+
+task :default => :test
+task compile: 'lib/jsgf/parser.rb'
+
+# Recompile the parser before building the gem package
+task :build => :compile
+
+Rake::TestTask.new(:test => :compile) do |t|
+    t.libs.push "lib"
+    t.test_files = FileList['test/**/*.rb']
+    t.verbose = true
+end

data/jsgf.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
     spec.name          = "jsgf"
-    spec.version       = '0'
+    spec.version       = '0.1'
     spec.authors       = ["Brandon Fosdick"]
     spec.email         = ["bfoz@bfoz.net"]
     spec.summary       = %q{Java Speech Grammar Format}
@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
     spec.homepage      = "http://github.com/bfoz/jsgf-ruby"
     spec.license       = "BSD"
 
-    spec.files         = `git ls-files -z`.split("\x0")
+    spec.files         = `git ls-files -z`.split("\x0").push('lib/jsgf/parser.rb').reject {|f| File.basename(f) == 'parser.y'}
     spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
     spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
     spec.require_paths = ["lib"]

data/lib/jsgf/alternation.rb

@@ -0,0 +1,25 @@
+module JSGF
+    class Alternation
+	attr_reader :elements
+	attr_writer :optional
+	attr_reader :tags
+
+	def initialize(*args)
+	    @elements = args
+	    @optional = false
+	    @tags = []
+	end
+
+	def push(*args)
+	    @elements.push *args
+	end
+
+	def size
+	    @elements.size
+	end
+
+	def weights
+	    @elements.map {|a| a[:weight]}
+	end
+    end
+end

data/lib/jsgf/grammar.rb

@@ -0,0 +1,23 @@
+module JSGF
+    class Grammar
+	attr_reader :character_encoding
+	attr_reader :locale
+	attr_reader :private_rules
+	attr_reader :public_rules
+	attr_reader :version
+
+	def initialize(character_encoding:nil, locale:nil, private_rules:{}, public_rules:{}, version:nil)
+	    @character_encoding = character_encoding
+	    @locale = locale
+	    @private_rules = private_rules
+	    @public_rules = public_rules
+	    @version = version
+	end
+
+	# @!attribute rules
+	#   @return [Hash]  Returns a {Hash} of the public and private rules, keyed by rule name
+	def rules
+	    @public_rules.merge(@private_rules)
+	end
+    end
+end

data/lib/jsgf/optional.rb

@@ -0,0 +1,11 @@
+module JSGF
+    class Optional
+	def initialize(*args)
+	    @elements = args
+	end
+
+	def optional
+	    true
+	end
+    end
+end

data/lib/jsgf/parser.rb

@@ -0,0 +1,390 @@
+#
+# DO NOT MODIFY!!!!
+# This file is automatically generated by Racc 1.4.12
+# from Racc grammer file "".
+#
+
+require 'racc/parser.rb'
+
+
+require_relative 'alternation'
+require_relative 'grammar'
+require_relative 'optional'
+require_relative 'repetition'
+
+module JSGF
+  class Parser < Racc::Parser
+
+
+attr_reader :grammar_name
+attr_reader :handler
+
+def initialize(tokenizer)
+    @private_rules = {}
+    @public_rules = {}
+    tokenizer = JSGF::Tokenizer.new(tokenizer) if tokenizer.is_a?(String)
+    @tokenizer = tokenizer
+    super()
+end
+
+def define_alternation(*args)
+    if args.first.is_a?(JSGF::Alternation)
+	args.first.push *(args.drop(2))
+	args.first
+    else
+	JSGF::Alternation.new(*(args.each_slice(2).map {|a,b| a}))
+    end
+end
+
+def define_optional(*args)
+    if args.first.respond_to?(:optional)
+	args.first.optional = true
+	args.first
+    else
+	JSGF::Optional.new(*args)
+    end
+end
+
+def define_atom(atom, weight=1.0, tags=[])
+   {atom:atom, weight:weight, tags:[]}
+end
+
+def define_rule(name, visibility=:private, *args)
+    r = {name:name, visibility:visibility, atoms:args}
+    if visibility == :private
+	@private_rules[name] = r
+    else
+	@public_rules[name] = r
+    end
+    r
+end
+
+def next_token
+    @tokenizer.next_token
+end
+
+def parse
+    do_parse
+    JSGF::Grammar.new(	character_encoding:@charset,
+			locale:@locale,
+			private_rules:@private_rules,
+			public_rules:@public_rules,
+			version:@version)
+end
+##### State transition tables begin ###
+
+racc_action_table = [
+    40,    37,    17,    11,    16,    40,    37,    38,    11,    39,
+    40,    37,    38,    11,    39,    40,    44,    38,    11,    39,
+    40,    37,    38,    11,    39,    40,    37,    38,    11,    39,
+    40,    37,    38,    11,    39,    40,    37,    38,    11,    39,
+     9,    13,    38,    45,    39,    13,    11,     9,    13,    13,
+    11,    49,    48,    11,    11,    61,    49,    62,    49,    51,
+    60,    49,    28,    47,    27,    46,    52,    53,    52,    53,
+    30,    29,    26,    11,    24,    58,    23,    51,    11,    18,
+    15,     5,     4 ]
+
+racc_action_check = [
+    59,    59,     4,    59,     4,    32,    32,    59,    32,    59,
+    24,    24,    32,    24,    32,    37,    25,    24,    37,    24,
+    38,    38,    37,    38,    37,    39,    39,    38,    39,    38,
+    49,    49,    39,    49,    39,    44,    44,    49,    44,    49,
+     2,     2,    44,    26,    44,     6,     2,     7,     7,    20,
+     6,    56,    31,     7,    20,    56,    31,    57,    55,    33,
+    55,    57,    17,    28,    17,    28,    35,    35,    54,    54,
+    23,    22,    15,    13,    12,    47,    11,    50,     9,     5,
+     3,     1,     0 ]
+
+racc_action_pointer = [
+    79,    81,    36,    78,    -5,    79,    40,    43,   nil,    68,
+   nil,    69,    62,    63,   nil,    65,   nil,    55,   nil,   nil,
+    44,   nil,    62,    59,     3,     4,    34,   nil,    56,   nil,
+   nil,    43,    -2,    53,   nil,    48,   nil,     8,    13,    18,
+   nil,   nil,   nil,   nil,    28,   nil,   nil,    66,   nil,    23,
+    71,   nil,   nil,   nil,    50,    45,    38,    48,   nil,    -7,
+   nil,   nil,   nil ]
+
+racc_action_default = [
+   -35,   -35,    -1,   -35,   -35,   -35,    -2,   -35,   -10,   -35,
+   -13,   -35,   -35,   -35,    -4,   -35,    -5,   -35,    63,   -14,
+    -3,   -11,   -35,   -35,   -35,   -35,   -35,    -6,   -35,   -12,
+   -15,   -35,   -18,   -20,   -22,   -24,   -25,   -35,   -35,   -35,
+   -29,   -30,   -31,   -32,   -35,    -9,    -7,   -35,   -16,   -35,
+   -21,   -23,   -33,   -34,   -26,   -35,   -35,   -35,    -8,   -19,
+   -27,   -28,   -17 ]
+
+racc_goto_table = [
+    50,    31,    12,    19,     1,     6,    12,    12,    14,    22,
+    20,     3,     7,    25,     8,    55,    56,    19,    59,    21,
+    12,    57,     2,    54,   nil,   nil,   nil,    50 ]
+
+racc_goto_check = [
+    12,    10,     8,     9,     1,     3,     8,     8,     6,     8,
+     3,     5,     4,     8,     7,    10,    10,     9,    11,     7,
+     8,    10,     2,    14,   nil,   nil,   nil,    12 ]
+
+racc_goto_pointer = [
+   nil,     4,    22,     3,    10,    11,     5,    12,     0,    -3,
+   -23,   -31,   -32,   nil,   -14,   nil,   nil,   nil ]
+
+racc_goto_default = [
+   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,    41,    10,
+   nil,    32,    33,    34,    35,    36,    42,    43 ]
+
+racc_reduce_table = [
+  0, 0, :racc_error,
+  1, 21, :_reduce_none,
+  2, 21, :_reduce_none,
+  3, 21, :_reduce_none,
+  2, 22, :_reduce_none,
+  2, 25, :_reduce_none,
+  3, 25, :_reduce_6,
+  4, 25, :_reduce_7,
+  5, 25, :_reduce_8,
+  3, 26, :_reduce_9,
+  1, 24, :_reduce_none,
+  2, 24, :_reduce_none,
+  3, 27, :_reduce_none,
+  1, 23, :_reduce_none,
+  2, 23, :_reduce_none,
+  3, 28, :_reduce_15,
+  4, 29, :_reduce_16,
+  5, 29, :_reduce_17,
+  1, 30, :_reduce_18,
+  3, 30, :_reduce_19,
+  1, 31, :_reduce_20,
+  2, 31, :_reduce_21,
+  1, 32, :_reduce_none,
+  2, 32, :_reduce_23,
+  1, 33, :_reduce_none,
+  1, 33, :_reduce_none,
+  2, 35, :_reduce_26,
+  3, 36, :_reduce_27,
+  3, 37, :_reduce_28,
+  1, 34, :_reduce_29,
+  1, 34, :_reduce_none,
+  1, 34, :_reduce_none,
+  1, 34, :_reduce_none,
+  2, 34, :_reduce_33,
+  2, 34, :_reduce_34 ]
+
+racc_reduce_n = 35
+
+racc_shift_n = 63
+
+racc_token_table = {
+  false => 0,
+  :error => 1,
+  :GRAMMAR => 2,
+  :HEADER => 3,
+  :IMPORT => 4,
+  :PUBLIC => 5,
+  :TAG => 6,
+  :TOKEN => 7,
+  :WEIGHT => 8,
+  ";" => 9,
+  "<" => 10,
+  ">" => 11,
+  "=" => 12,
+  "|" => 13,
+  "(" => 14,
+  ")" => 15,
+  "[" => 16,
+  "]" => 17,
+  "*" => 18,
+  "+" => 19 }
+
+racc_nt_base = 20
+
+racc_use_result_var = true
+
+Racc_arg = [
+  racc_action_table,
+  racc_action_check,
+  racc_action_default,
+  racc_action_pointer,
+  racc_goto_table,
+  racc_goto_check,
+  racc_goto_default,
+  racc_goto_pointer,
+  racc_nt_base,
+  racc_reduce_table,
+  racc_token_table,
+  racc_shift_n,
+  racc_reduce_n,
+  racc_use_result_var ]
+
+Racc_token_to_s_table = [
+  "$end",
+  "error",
+  "GRAMMAR",
+  "HEADER",
+  "IMPORT",
+  "PUBLIC",
+  "TAG",
+  "TOKEN",
+  "WEIGHT",
+  "\";\"",
+  "\"<\"",
+  "\">\"",
+  "\"=\"",
+  "\"|\"",
+  "\"(\"",
+  "\")\"",
+  "\"[\"",
+  "\"]\"",
+  "\"*\"",
+  "\"+\"",
+  "$start",
+  "grammar",
+  "header",
+  "rule_list",
+  "import_header",
+  "jsgf_header",
+  "grammar_header",
+  "import_statement",
+  "rule_name",
+  "rule",
+  "alternate_list",
+  "rule_expansion",
+  "tagged_rule_item",
+  "rule_item",
+  "rule_atom",
+  "weighted_item",
+  "rule_group",
+  "rule_optional" ]
+
+Racc_debug_parser = false
+
+##### State transition tables end #####
+
+# reduce 0 omitted
+
+# reduce 1 omitted
+
+# reduce 2 omitted
+
+# reduce 3 omitted
+
+# reduce 4 omitted
+
+# reduce 5 omitted
+
+def _reduce_6(val, _values, result)
+ @version = val[1]; 
+    result
+end
+
+def _reduce_7(val, _values, result)
+ @version = val[1]; @charset = val[2]; 
+    result
+end
+
+def _reduce_8(val, _values, result)
+ @version = val[1]; @charset = val[2]; @locale = val[3]; 
+    result
+end
+
+def _reduce_9(val, _values, result)
+ @grammar_name = val[1] 
+    result
+end
+
+# reduce 10 omitted
+
+# reduce 11 omitted
+
+# reduce 12 omitted
+
+# reduce 13 omitted
+
+# reduce 14 omitted
+
+def _reduce_15(val, _values, result)
+ result = val[1] 
+    result
+end
+
+def _reduce_16(val, _values, result)
+ define_rule(val[0], :private, *(val[2..-2])) 
+    result
+end
+
+def _reduce_17(val, _values, result)
+ define_rule(val[1], :public, val[3]) 
+    result
+end
+
+def _reduce_18(val, _values, result)
+ result = val.first 
+    result
+end
+
+def _reduce_19(val, _values, result)
+ result = define_alternation(*val) 
+    result
+end
+
+def _reduce_20(val, _values, result)
+ result = val.first 
+    result
+end
+
+def _reduce_21(val, _values, result)
+ result = val 
+    result
+end
+
+# reduce 22 omitted
+
+def _reduce_23(val, _values, result)
+ val[0][:tags].push(val[1]); result = val[0] 
+    result
+end
+
+# reduce 24 omitted
+
+# reduce 25 omitted
+
+def _reduce_26(val, _values, result)
+ val[1][:weight] = val.first[1..-2].to_f; result = val[1] 
+    result
+end
+
+def _reduce_27(val, _values, result)
+ result = val[1] 
+    result
+end
+
+def _reduce_28(val, _values, result)
+ result = define_optional(val[1]) 
+    result
+end
+
+def _reduce_29(val, _values, result)
+ result = define_atom(val.first) 
+    result
+end
+
+# reduce 30 omitted
+
+# reduce 31 omitted
+
+# reduce 32 omitted
+
+def _reduce_33(val, _values, result)
+ result = JSGF::Repetition.new(val[0], 0) 
+    result
+end
+
+def _reduce_34(val, _values, result)
+ result = JSGF::Repetition.new(val[0], 1) 
+    result
+end
+
+def _reduce_none(val, _values, result)
+  val[0]
+end
+
+  end   # class Parser
+  end   # module JSGF

data/lib/jsgf/repetition.rb

@@ -0,0 +1,10 @@
+module JSGF
+    class Repetition
+	# @param atom [Hash]	the atom to repeat
+	# @param minimum_count	[Number]	the minimum repetition count (0 or 1)
+	def initialize(atom, minimum_count)
+	    @atom = atom
+	    @count = minimum_count
+	end
+    end
+end

data/lib/jsgf/tokenizer.rb

@@ -0,0 +1,35 @@
+require 'strscan'
+
+module JSGF
+    class Tokenizer
+	TOKENS = {
+	    /grammar/				=> :GRAMMAR,
+	    /#JSGF/				=> :HEADER,
+	    /import/				=> :IMPORT,
+	    /public/				=> :PUBLIC,
+	    /\{(\\.|[^\}]+)*\}/			=> :TAG,
+	    /[^ \t\n;=|*+<>\(\)\[\]{}*\/]+/	=> :TOKEN,
+	    %r{/\d*(\.\d+)?/}			=> :WEIGHT,
+	}
+
+	# @param io [IO]    the {IO} stream to read from
+	def initialize(io)
+	    io = io.read unless io.is_a?(String)
+	    @scanner = StringScanner.new(io)
+	end
+
+	def next_token
+	    return if @scanner.eos?
+
+	    @scanner.skip(/\s+/)	# Skip all leading whitespace
+	    @scanner.skip(%r{//.*\n})	# Skip single-line comments
+
+	    TOKENS.each do |(pattern, token)|
+		text = @scanner.scan(pattern)
+		return [token, text] if text
+	    end
+	    x = @scanner.getch
+	    [x, x] unless x.nil?
+	end
+    end
+end

data/lib/jsgf.rb

@@ -1,3 +1,19 @@
-module Jsgf
-  # Your code goes here...
+require_relative 'jsgf/parser'
+require_relative 'jsgf/tokenizer'
+
+=begin
+A parser for JSGF files.
+
+http://www.w3.org/TR/jsgf/
+
+# Usage
+    grammar = JSGF.read(filename)
+=end
+module JSGF
+    # @param filename	[String]    the file to parse
+    # @return [Grammar]	the resulting {Grammar}
+    def self.read(filename)
+	tokenzier = JSGF::Tokenizer.new(File.read(filename))
+	JSGF::Parser.new(tokenzier).parse
+    end
 end

data/test/jsgf/parser.rb

@@ -0,0 +1,109 @@
+require 'minitest/autorun'
+require 'jsgf/parser'
+
+describe JSGF::Parser do
+    describe 'when parsing the header' do
+	it 'must parse the version' do
+	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;').parse
+	    grammar.version.must_equal 'V1.0'
+	end
+
+	it 'must parse the character encoding' do
+	    grammar = JSGF::Parser.new('#JSGF V1.0 CHARSET; grammar header_grammar;').parse
+	    grammar.version.must_equal 'V1.0'
+	    grammar.character_encoding.must_equal 'CHARSET'
+	end
+
+	it 'must parse the locale' do
+	    grammar = JSGF::Parser.new('#JSGF V1.0 CHARSET locale; grammar header_grammar;').parse
+	    grammar.version.must_equal 'V1.0'
+	    grammar.character_encoding.must_equal 'CHARSET'
+	    grammar.locale.must_equal 'locale'
+	end
+    end
+
+    it 'must parse a rule with a single atom' do
+	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one;').parse
+	grammar.rules.size.must_equal 1
+	grammar.rules.keys.must_equal ['rule']
+	grammar.rules['rule'][:atoms].size.must_equal 1
+    end
+
+    it 'must parse a rule with multiple atoms' do
+	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one two;').parse
+	grammar.rules.size.must_equal 1
+	grammar.rules.keys.must_equal ['rule']
+	grammar.rules['rule'][:atoms].size.must_equal 1
+	grammar.rules['rule'][:atoms].first.size.must_equal 2
+    end
+
+    it 'must parse a rule with multiple grouped atoms' do
+	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=(one two);').parse
+	grammar.rules.size.must_equal 1
+	grammar.rules.keys.must_equal ['rule']
+	grammar.rules['rule'][:atoms].size.must_equal 1
+	grammar.rules['rule'][:atoms].first.size.must_equal 2
+    end
+
+    describe 'when parsing an alternation' do
+	it 'must parse a rule with an alternation' do
+	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one | two;').parse
+	    grammar.rules.size.must_equal 1
+	    grammar.rules.keys.must_equal ['rule']
+	    grammar.rules['rule'][:atoms].size.must_equal 1
+
+	    alternation = grammar.rules['rule'][:atoms].first
+	    alternation.must_be_kind_of JSGF::Alternation
+	    alternation.size.must_equal 2
+	end
+	
+	it 'must parse a rule with a bigger alternation' do
+	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=one | two | three;').parse
+	    grammar.rules.size.must_equal 1
+	    grammar.rules.keys.must_equal ['rule']
+	    grammar.rules['rule'][:atoms].size.must_equal 1
+
+	    alternation = grammar.rules['rule'][:atoms].first
+	    alternation.must_be_kind_of JSGF::Alternation
+	    alternation.size.must_equal 3
+	    alternation.elements[0][:atom].must_equal 'one'
+	    alternation.elements[1][:atom].must_equal 'two'
+	    alternation.elements[2][:atom].must_equal 'three'
+	end
+
+	it 'must parse a weighted alternation' do
+	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>= /0.5/ one | /0.25/ two | /0.25/ three;').parse
+	    grammar.rules.keys.must_equal ['rule']
+	    grammar.rules['rule'][:atoms].size.must_equal 1
+
+	    alternation = grammar.rules['rule'][:atoms].first
+	    alternation.must_be_kind_of JSGF::Alternation
+	    alternation.size.must_equal 3
+	    alternation.weights.must_equal [0.5, 0.25, 0.25]
+	    alternation.elements[0][:atom].must_equal 'one'
+	end
+
+	it 'must parse a grouped alternation' do
+	    grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=(one | two | three);').parse
+	    grammar.rules.size.must_equal 1
+	    grammar.rules.keys.must_equal ['rule']
+	    grammar.rules['rule'][:atoms].size.must_equal 1
+	    grammar.rules['rule'][:atoms].first.must_be_kind_of JSGF::Alternation
+	    grammar.rules['rule'][:atoms].first.size.must_equal 3
+	end
+    end
+
+    it 'must parse an optional group of a single atom' do
+	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=[one];').parse
+	grammar.rules.size.must_equal 1
+	grammar.rules.keys.must_equal ['rule']
+	grammar.rules['rule'][:atoms].size.must_equal 1
+    end
+
+    it 'must parse an optional alternation' do
+	grammar = JSGF::Parser.new('#JSGF V1.0; grammar header_grammar;<rule>=[one | two];').parse
+	grammar.rules.size.must_equal 1
+	grammar.rules.keys.must_equal ['rule']
+	grammar.rules['rule'][:atoms].size.must_equal 1
+    end
+end

data/test/jsgf/tokenizer.rb

@@ -0,0 +1,36 @@
+require 'minitest/autorun'
+require 'jsgf/tokenizer'
+
+describe JSGF::Tokenizer do
+    it 'must pass individual characters' do
+	JSGF::Tokenizer.new(';').next_token.must_equal [';', ';']
+    end
+
+    it 'must recognize the grammar keyword' do
+	JSGF::Tokenizer.new('grammar').next_token.must_equal [:GRAMMAR, 'grammar']
+    end
+
+    it 'must skip leading whitespace' do
+	JSGF::Tokenizer.new('   grammar').next_token.must_equal [:GRAMMAR, 'grammar']
+    end
+
+    it 'must recognize the header string' do
+	JSGF::Tokenizer.new('#JSGF').next_token.must_equal [:HEADER, '#JSGF']
+    end
+
+    it 'must recognize the public keyword' do
+	JSGF::Tokenizer.new('public').next_token.must_equal [:PUBLIC, 'public']
+    end
+
+    it 'must recognize a tag block ' do
+	JSGF::Tokenizer.new('{ some stuff }').next_token.must_equal [:TAG, '{ some stuff }']
+    end
+
+    it 'must recognize a weight' do
+	JSGF::Tokenizer.new('/0.5/').next_token.must_equal [:WEIGHT, '/0.5/']
+    end
+
+    it 'must recognize a token' do
+	JSGF::Tokenizer.new(' some_token ').next_token.must_equal [:TOKEN, 'some_token']
+    end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jsgf
 version: !ruby/object:Gem::Version
-  version: '0'
+  version: '0.1'
 platform: ruby
 authors:
 - Brandon Fosdick
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-02 00:00:00.000000000 Z
+date: 2015-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -51,6 +51,14 @@ files:
 - Rakefile
 - jsgf.gemspec
 - lib/jsgf.rb
+- lib/jsgf/alternation.rb
+- lib/jsgf/grammar.rb
+- lib/jsgf/optional.rb
+- lib/jsgf/parser.rb
+- lib/jsgf/repetition.rb
+- lib/jsgf/tokenizer.rb
+- test/jsgf/parser.rb
+- test/jsgf/tokenizer.rb
 homepage: http://github.com/bfoz/jsgf-ruby
 licenses:
 - BSD
@@ -75,4 +83,6 @@ rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Java Speech Grammar Format
-test_files: []
+test_files:
+- test/jsgf/parser.rb
+- test/jsgf/tokenizer.rb