syntax 1.1.0 → 1.2.0

data/test/syntax/tc_xml.rb

@@ -1,202 +0,0 @@
-$:.unshift File.dirname(__FILE__) +"/../../lib"
-
-require 'test/unit'
-require 'syntax/lang/xml'
-
-class TC_Syntax_XML < Test::Unit::TestCase
-
-  def setup
-    @xml = Syntax::XML.new
-  end
-
-  def test_empty
-    called = false
-    @xml.tokenize( "" ) { |tok| called = true }
-    assert !called
-  end
-
-  def test_no_tag
-    tok = []
-    @xml.tokenize( "foo bar baz" ) { |t| tok << t }
-    assert_equal [ :normal, "foo bar baz" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_entity_outside_tag
-    tok = []
-    @xml.tokenize( "&amp; &#10; &x157; &nosemi & foo;" ) { |t| tok << t }
-    assert_equal [ :entity, "&amp;" ], [ tok.first.group, tok.shift ]
-    tok.shift
-    assert_equal [ :entity, "&#10;" ], [ tok.first.group, tok.shift ]
-    tok.shift
-    assert_equal [ :entity, "&x157;" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " &nosemi & foo;" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_start_tag
-    tok = []
-    @xml.tokenize( "<name" ) { |t| tok << t }
-    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_start_xml_decl
-    tok = []
-    @xml.tokenize( "<?xml" ) { |t| tok << t }
-    assert_equal [ :punct, "<?" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :tag, "xml" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_start_end_tag
-    tok = []
-    @xml.tokenize( "</name" ) { |t| tok << t }
-    assert_equal [ :punct, "</" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_start_decl_tag
-    tok = []
-    @xml.tokenize( "<!name" ) { |t| tok << t }
-    assert_equal [ :punct, "<!" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_start_confused
-    tok = []
-    @xml.tokenize( "<%name" ) { |t| tok << t }
-    assert_equal [ :punct, "<%" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :attribute, "name" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_end_tag_out_of_context
-    tok = []
-    @xml.tokenize( "/>" ) { |t| tok << t }
-    assert_equal [ :normal, "/>" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_start_namespaced_tag
-    tok = []
-    @xml.tokenize( "<foo:name" ) { |t| tok << t }
-    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :namespace, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_attribute
-    tok = []
-    @xml.tokenize( "<name attr1 attr2" ) { |t| tok << t }
-    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :attribute, "attr1" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :attribute, "attr2" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_namespaced_attribute
-    tok = []
-    @xml.tokenize( "<name foo:attr1 bar:attr2" ) { |t| tok << t }
-    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :namespace, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :attribute, "attr1" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :namespace, "bar" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :attribute, "attr2" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_attribute_with_squote_value
-    tok = []
-    @xml.tokenize( "<name attr1='a value < > \\' here'" ) { |t| tok << t }
-    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :attribute, "attr1" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "='" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, "a value < > \\' here" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_attribute_with_dquote_value
-    tok = []
-    @xml.tokenize( '<name attr1="a value < > \" here"' ) { |t| tok << t }
-    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :attribute, "attr1" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, '="' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, 'a value < > \" here' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_entity_in_string
-    tok = []
-    @xml.tokenize( '<name "a &lt; value &gt;"' ) { |t| tok << t }
-    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, "a " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :entity, "&lt;" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, " value " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :entity, "&gt;" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_number
-    tok = []
-    @xml.tokenize( '<name 5' ) { |t| tok << t }
-    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :number, "5" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_close_tag
-    tok = []
-    @xml.tokenize( '<name> foo' ) { |t| tok << t }
-    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ">" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " foo" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_close_self_tag
-    tok = []
-    @xml.tokenize( '<name /> foo' ) { |t| tok << t }
-    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "/>" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " foo" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_close_decl_tag
-    tok = []
-    @xml.tokenize( '<?name ?> foo' ) { |t| tok << t }
-    assert_equal [ :punct, "<?" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "?>" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " foo" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_comment
-    tok = []
-    @xml.tokenize( "foo <!-- a comment\nspanning multiple\nlines --> bar" ) { |t| tok << t }
-    assert_equal [ :normal, "foo " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :comment, "<!-- a comment\nspanning multiple\nlines -->" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " bar" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_comment_unterminated
-    tok = []
-    @xml.tokenize( "foo <!-- a comment\nspanning multiple\nlines -- bar" ) { |t| tok << t }
-    assert_equal [ :normal, "foo " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :comment, "<!-- a comment\nspanning multiple\nlines -- bar" ], [ tok.first.group, tok.shift ]
-  end
-
-end

data/test/syntax/tc_yaml.rb

@@ -1,228 +0,0 @@
-$:.unshift File.dirname(__FILE__) +"/../../lib"
-
-require 'test/unit'
-require 'syntax/lang/yaml'
-
-class TC_Syntax_YAML < Test::Unit::TestCase
-
-  def setup
-    @yaml = Syntax::YAML.new
-  end
-
-  def test_empty
-    called = false
-    @yaml.tokenize( "" ) { |tok| called = true }
-    assert !called
-  end
-
-  def test_doc_notype
-    tok = []
-    @yaml.tokenize( "---\n" ) { |t| tok << t }
-    assert_equal [ :document, "---" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_doc_type
-    tok = []
-    @yaml.tokenize( "--- !foo/bar/^type \n" ) { |t| tok << t }
-    assert_equal [ :document, "--- !foo/bar/^type " ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_key_no_indent
-    tok = []
-    @yaml.tokenize( "foo : bar" ) { |t| tok << t }
-    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " bar" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_key_indent
-    tok = []
-    @yaml.tokenize( "  foo : bar" ) { |t| tok << t }
-    assert_equal [ :normal, "  " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " bar" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_key_quoted
-    tok = []
-    @yaml.tokenize( "  'foo' : bar" ) { |t| tok << t }
-    assert_equal [ :normal, "  " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " bar" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_key_no_value
-    tok = []
-    @yaml.tokenize( "  foo: \n  bar:\n" ) { |t| tok << t }
-    assert_equal [ :normal, "  " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " \n  " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :key, "bar" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_list_no_indent
-    tok = []
-    @yaml.tokenize( "- bar" ) { |t| tok << t }
-    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " bar" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_list_indent
-    tok = []
-    @yaml.tokenize( "  - bar" ) { |t| tok << t }
-    assert_equal [ :normal, "  " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " bar" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_blank_lines
-    tok = []
-    @yaml.tokenize( "foo: bar\n\n\nbaz: bang" ) { |t| tok << t }
-    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " bar\n\n\n" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :key, "baz" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " bang" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_type
-    tok = []
-    @yaml.tokenize( "  - !name/space^type\n    - foo" ) { |t| tok << t }
-    assert_equal [ :normal, "  " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :type, "!name/space^type" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, "\n    " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " foo" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_anchor_ref
-    tok = []
-    @yaml.tokenize( "foo: &blah\nbar: *blah" ) { |t| tok << t }
-    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :anchor, "&blah" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :key, "bar" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :ref, "*blah" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_time
-    tok = []
-    @yaml.tokenize( "foo: 01:23:45\n" ) { |t| tok << t }
-    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :time, "01:23:45" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_date
-    tok = []
-    @yaml.tokenize( "foo: 1234-56-78 01:23:45.123456 +01:23\n" ) { |t| tok << t }
-    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :date, "1234-56-78 01:23:45.123456 +01:23" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_string_dquote
-    tok = []
-    @yaml.tokenize( 'foo: "this is a \"string\""' ) { |t| tok << t }
-    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, 'this is a ' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :expr, '\"' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, 'string' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :expr, '\"' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_string_squote
-    tok = []
-    @yaml.tokenize( "foo: 'this is a \\\"string\\\"'" ) { |t| tok << t }
-    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, 'this is a \"string\"' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_symbol
-    tok = []
-    @yaml.tokenize( "  :foo: 'this is a \\\"string\\\"'" ) { |t| tok << t }
-    assert_equal [ :normal, "  " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :symbol, ":foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_comment
-    tok = []
-    @yaml.tokenize( "foo: a value # comment\n# another comment: foo\n" ) { |t| tok << t }
-    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " a value " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :comment, "# comment" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :comment, "# another comment: foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_long_text
-    tok = []
-    text = <<EOF
-foo: >-
-  a b c d
-  e f g
-
-  h i
-
-
-  j k l
-  m n
-
-bar: baz
-EOF
-    @yaml.tokenize( text ) { |t| tok << t }
-    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ">-" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, "  a b c d\n  e f g\n\n  h i\n\n\n  j k l\n  m n\n\n" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :key, "bar" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " baz\n" ], [ tok.first.group, tok.shift ]
-  end
-
-  def test_long_test_at_eof
-    tok = []
-    @yaml.tokenize( "foo: >\n  one two\n  three four" ) { |t| tok << t }
-    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, ">" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, "  one two\n  three four" ], [ tok.first.group, tok.shift ]
-  end
-
-end

data/test/syntax/tokenizer_testcase.rb

@@ -1,40 +0,0 @@
-$:.unshift File.dirname(__FILE__) + "/../../lib"
-
-require 'test/unit'
-require 'syntax'
-
-class TokenizerTestCase < Test::Unit::TestCase
-  def self.syntax( type )
-    class_eval <<-EOF
-      def setup
-        @tokenizer = Syntax.load(#{type.inspect})
-      end
-    EOF
-  end
-
-  def default_test
-  end
-
-  private
-
-    attr_reader :tokenizer
-
-    def tokenize( string )
-      @tokens = []
-      @tokenizer.tokenize( string ) { |tok| @tokens << tok }
-    end
-
-    def assert_next_token(group, lexeme, instruction=:none)
-      assert false, "no tokens in stack" if @tokens.nil? or @tokens.empty?
-      assert_equal [group, lexeme, instruction],
-        [@tokens.first.group, @tokens.first, @tokens.shift.instruction]
-    end
-
-    def assert_no_next_token
-      assert @tokens.empty?
-    end
-
-    def skip_token( n=1 )
-      n.times { @tokens.shift } unless @tokens.nil? || @tokens.empty?
-    end
-end