syntax 0.5.0

data/test/syntax/tc_xml.rb

@@ -0,0 +1,202 @@
+$:.unshift "../../lib"
+
+require 'test/unit'
+require 'syntax/xml'
+
+class TC_Syntax_XML < Test::Unit::TestCase
+
+  def setup
+    @xml = Syntax::XML.new
+  end
+
+  def test_empty
+    called = false
+    @xml.tokenize( "" ) { |tok| called = true }
+    assert !called
+  end
+
+  def test_no_tag
+    tok = []
+    @xml.tokenize( "foo bar baz" ) { |t| tok << t }
+    assert_equal [ :normal, "foo bar baz" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_entity_outside_tag
+    tok = []
+    @xml.tokenize( "&amp; &#10; &x157; &nosemi & foo;" ) { |t| tok << t }
+    assert_equal [ :entity, "&amp;" ], [ tok.first.group, tok.shift ]
+    tok.shift
+    assert_equal [ :entity, "&#10;" ], [ tok.first.group, tok.shift ]
+    tok.shift
+    assert_equal [ :entity, "&x157;" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " &nosemi & foo;" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_start_tag
+    tok = []
+    @xml.tokenize( "<name" ) { |t| tok << t }
+    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_start_xml_decl
+    tok = []
+    @xml.tokenize( "<?xml" ) { |t| tok << t }
+    assert_equal [ :punct, "<?" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :tag, "xml" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_start_end_tag
+    tok = []
+    @xml.tokenize( "</name" ) { |t| tok << t }
+    assert_equal [ :punct, "</" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_start_decl_tag
+    tok = []
+    @xml.tokenize( "<!name" ) { |t| tok << t }
+    assert_equal [ :punct, "<!" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_start_confused
+    tok = []
+    @xml.tokenize( "<%name" ) { |t| tok << t }
+    assert_equal [ :punct, "<%" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :attribute, "name" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_end_tag_out_of_context
+    tok = []
+    @xml.tokenize( "/>" ) { |t| tok << t }
+    assert_equal [ :normal, "/>" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_start_namespaced_tag
+    tok = []
+    @xml.tokenize( "<foo:name" ) { |t| tok << t }
+    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :namespace, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_attribute
+    tok = []
+    @xml.tokenize( "<name attr1 attr2" ) { |t| tok << t }
+    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :attribute, "attr1" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :attribute, "attr2" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_namespaced_attribute
+    tok = []
+    @xml.tokenize( "<name foo:attr1 bar:attr2" ) { |t| tok << t }
+    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :namespace, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :attribute, "attr1" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :namespace, "bar" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :attribute, "attr2" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_attribute_with_squote_value
+    tok = []
+    @xml.tokenize( "<name attr1='a value < > \\' here'" ) { |t| tok << t }
+    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :attribute, "attr1" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "='" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, "a value < > \\' here" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_attribute_with_dquote_value
+    tok = []
+    @xml.tokenize( '<name attr1="a value < > \" here"' ) { |t| tok << t }
+    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :attribute, "attr1" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, '="' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, 'a value < > \" here' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_entity_in_string
+    tok = []
+    @xml.tokenize( '<name "a &lt; value &gt;"' ) { |t| tok << t }
+    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, "a " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :entity, "&lt;" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, " value " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :entity, "&gt;" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_number
+    tok = []
+    @xml.tokenize( '<name 5' ) { |t| tok << t }
+    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :number, "5" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_close_tag
+    tok = []
+    @xml.tokenize( '<name> foo' ) { |t| tok << t }
+    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ">" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " foo" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_close_self_tag
+    tok = []
+    @xml.tokenize( '<name /> foo' ) { |t| tok << t }
+    assert_equal [ :punct, "<" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "/>" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " foo" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_close_decl_tag
+    tok = []
+    @xml.tokenize( '<?name ?> foo' ) { |t| tok << t }
+    assert_equal [ :punct, "<?" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :tag, "name" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "?>" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " foo" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_comment
+    tok = []
+    @xml.tokenize( "foo <!-- a comment\nspanning multiple\nlines --> bar" ) { |t| tok << t }
+    assert_equal [ :normal, "foo " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :comment, "<!-- a comment\nspanning multiple\nlines -->" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " bar" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_comment_unterminated
+    tok = []
+    @xml.tokenize( "foo <!-- a comment\nspanning multiple\nlines -- bar" ) { |t| tok << t }
+    assert_equal [ :normal, "foo " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :comment, "<!-- a comment\nspanning multiple\nlines -- bar" ], [ tok.first.group, tok.shift ]
+  end
+
+end

data/test/syntax/tc_yaml.rb

@@ -0,0 +1,228 @@
+$:.unshift "../../lib"
+
+require 'test/unit'
+require 'syntax/yaml'
+
+class TC_Syntax_YAML < Test::Unit::TestCase
+
+  def setup
+    @yaml = Syntax::YAML.new
+  end
+
+  def test_empty
+    called = false
+    @yaml.tokenize( "" ) { |tok| called = true }
+    assert !called
+  end
+
+  def test_doc_notype
+    tok = []
+    @yaml.tokenize( "---\n" ) { |t| tok << t }
+    assert_equal [ :document, "---" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_doc_type
+    tok = []
+    @yaml.tokenize( "--- !foo/bar/^type \n" ) { |t| tok << t }
+    assert_equal [ :document, "--- !foo/bar/^type " ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_key_no_indent
+    tok = []
+    @yaml.tokenize( "foo : bar" ) { |t| tok << t }
+    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " bar" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_key_indent
+    tok = []
+    @yaml.tokenize( "  foo : bar" ) { |t| tok << t }
+    assert_equal [ :normal, "  " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " bar" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_key_quoted
+    tok = []
+    @yaml.tokenize( "  'foo' : bar" ) { |t| tok << t }
+    assert_equal [ :normal, "  " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " bar" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_key_no_value
+    tok = []
+    @yaml.tokenize( "  foo: \n  bar:\n" ) { |t| tok << t }
+    assert_equal [ :normal, "  " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " \n  " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :key, "bar" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_list_no_indent
+    tok = []
+    @yaml.tokenize( "- bar" ) { |t| tok << t }
+    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " bar" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_list_indent
+    tok = []
+    @yaml.tokenize( "  - bar" ) { |t| tok << t }
+    assert_equal [ :normal, "  " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " bar" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_blank_lines
+    tok = []
+    @yaml.tokenize( "foo: bar\n\n\nbaz: bang" ) { |t| tok << t }
+    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " bar\n\n\n" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :key, "baz" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " bang" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_type
+    tok = []
+    @yaml.tokenize( "  - !name/space^type\n    - foo" ) { |t| tok << t }
+    assert_equal [ :normal, "  " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :type, "!name/space^type" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, "\n    " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " foo" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_anchor_ref
+    tok = []
+    @yaml.tokenize( "foo: &blah\nbar: *blah" ) { |t| tok << t }
+    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :anchor, "&blah" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :key, "bar" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :ref, "*blah" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_time
+    tok = []
+    @yaml.tokenize( "foo: 01:23:45\n" ) { |t| tok << t }
+    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :time, "01:23:45" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_date
+    tok = []
+    @yaml.tokenize( "foo: 1234-56-78 01:23:45.123456 +01:23\n" ) { |t| tok << t }
+    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :date, "1234-56-78 01:23:45.123456 +01:23" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_string_dquote
+    tok = []
+    @yaml.tokenize( 'foo: "this is a \"string\""' ) { |t| tok << t }
+    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, 'this is a ' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :expr, '\"' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, 'string' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :expr, '\"' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_string_squote
+    tok = []
+    @yaml.tokenize( "foo: 'this is a \\\"string\\\"'" ) { |t| tok << t }
+    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, 'this is a \"string\"' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_symbol
+    tok = []
+    @yaml.tokenize( "  :foo: 'this is a \\\"string\\\"'" ) { |t| tok << t }
+    assert_equal [ :normal, "  " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :symbol, ":foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_comment
+    tok = []
+    @yaml.tokenize( "foo: a value # comment\n# another comment: foo\n" ) { |t| tok << t }
+    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " a value " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :comment, "# comment" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :comment, "# another comment: foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_long_text
+    tok = []
+    text = <<EOF
+foo: >-
+  a b c d
+  e f g
+
+  h i
+
+
+  j k l
+  m n
+
+bar: baz
+EOF
+    @yaml.tokenize( text ) { |t| tok << t }
+    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ">-" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, "  a b c d\n  e f g\n\n  h i\n\n\n  j k l\n  m n\n\n" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :key, "bar" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " baz\n" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_long_test_at_eof
+    tok = []
+    @yaml.tokenize( "foo: >\n  one two\n  three four" ) { |t| tok << t }
+    assert_equal [ :key, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ":" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, ">" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, "\n" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, "  one two\n  three four" ], [ tok.first.group, tok.shift ]
+  end
+
+end

metadata

@@ -0,0 +1,51 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.4
+specification_version: 1
+name: syntax
+version: !ruby/object:Gem::Version 
+  version: 0.5.0
+date: 2005-01-11
+summary: Syntax is Ruby library for performing simple syntax highlighting.
+require_paths: 
+  - lib
+email: jgb3@email.byu.edu
+homepage: 
+rubyforge_project: 
+description: 
+autorequire: syntax
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+    - 
+      - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+  version: 
+platform: ruby
+authors: 
+  - Jamis Buck
+files: 
+  - lib/syntax.rb
+  - lib/syntax
+  - lib/syntax/yaml.rb
+  - lib/syntax/common.rb
+  - lib/syntax/ruby.rb
+  - lib/syntax/convertors
+  - lib/syntax/xml.rb
+  - lib/syntax/version.rb
+  - lib/syntax/convertors/html.rb
+  - test/ALL-TESTS.rb
+  - test/syntax
+  - test/syntax/tc_yaml.rb
+  - test/syntax/tc_ruby.rb
+  - test/syntax/tc_xml.rb
+test_files: 
+  - test/ALL-TESTS.rb
+rdoc_options: []
+extra_rdoc_files: []
+executables: []
+extensions: []
+requirements: []
+dependencies: []