lingo 1.8.5 → 1.8.6

data/test/attendee/ts_text_writer.rb

@@ -20,7 +20,8 @@ class TestAttendeeTextWriter < AttendeeTestCase
       wd('Zeile|IDF'),
       tk('.|PUNC'),
       ai('EOL|test/test.txt'),
-      ai('EOF|test/test.txt')
+      ai('EOF|test/test.txt'),
+      ai('EOT|')
     ]
   end
 
@@ -50,7 +51,7 @@ class TestAttendeeTextWriter < AttendeeTestCase
 
   def test_lir_file
     meet({ 'ext' => 'vec', 'lir-format' => false }, [
-      ai('LIR-FORMAT|'), ai('FILE|test/lir.txt'),
+      ai('LIR|'), ai('FILE|test/lir.txt'),
       ai('RECORD|00237'),
       '020: GERHARD.',
       '025: Automatisches Sammeln, Klassifizieren und Indexieren von wissenschaftlich relevanten Informationsressourcen.',
@@ -61,7 +62,8 @@ class TestAttendeeTextWriter < AttendeeTestCase
       ai('RECORD|00239'),
       '020: Information Retrieval und Dokumentmanagement im Multimedia-Zeitalter.',
       "056: \"Das Buch ist ein praxisbezogenes VADEMECUM für alle, die in einer Welt der Datennetze Wissen/Informationen sammeln.",
-      ai('EOF|test/lir.txt')
+      ai('EOF|test/lir.txt'),
+      ai('EOT|')
     ])
 
     assert_equal([
@@ -76,7 +78,7 @@ FG-Projekt GERHARD.\n",
 
   def test_nonewords
     meet({ 'ext' => 'non', 'sep' => "\n" }, [
-      ai('FILE|test/text.txt'), 'Nonwörter', 'Nonsense', ai('EOF|test/text.txt')
+      ai('FILE|test/text.txt'), 'Nonwörter', 'Nonsense', ai('EOF|test/text.txt'), ai('EOT|')
     ])
 
     assert_equal([

data/test/attendee/ts_tokenizer.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 
+require_relative '../test_helper'
+
 class TestAttendeeTokenizer < AttendeeTestCase
 
   def setup
@@ -19,13 +21,24 @@ class TestAttendeeTokenizer < AttendeeTestCase
       '',
       '}}'
     ]
+
+    @html = [
+      'test <a>test</a> test',
+      '<b>test <a>test</a></b>',
+      'test <a test="test"><b>test</b></a>, test',
+      '<a>test</a><b test="test">test</b><a>test</a>'
+    ]
   end
 
   def test_basic
     meet({}, [
       'Dies ist ein Test.'
     ], [
-      tk('Dies|WORD'), tk('ist|WORD'), tk('ein|WORD'), tk('Test|WORD'), tk('.|PUNC')
+      tk('Dies|WORD|0|0'),
+      tk('ist|WORD|1|5'),
+      tk('ein|WORD|2|9'),
+      tk('Test|WORD|3|13'),
+      tk('.|PUNC|4|17')
     ])
   end
 
@@ -33,208 +46,306 @@ class TestAttendeeTokenizer < AttendeeTestCase
     meet({}, [
       '1964 www.vorhauer.de bzw. nasenbär, ()'
     ], [
-      tk('1964|NUMS'),
-      tk('www.vorhauer.de|URLS'),
-      tk('bzw|WORD'),
-      tk('.|PUNC'),
-      tk('nasenbär|WORD'),
-      tk(',|PUNC'),
-      tk('(|OTHR'),
-      tk(')|OTHR')
+      tk('1964|NUMS|0|0'),
+      tk('www.vorhauer.de|URLS|1|5'),
+      tk('bzw|WORD|2|21'),
+      tk('.|PUNC|3|24'),
+      tk('nasenbär|WORD|4|26'),
+      tk(',|PUNC|5|35'),
+      tk('(|OTHR|6|37'),
+      tk(')|OTHR|7|38')
     ])
   end
 
   def test_wiki1
     meet({}, @wiki, [
-      tk('Test|WORD'),
-      tk('[|OTHR'),
-      tk('[|OTHR'),
-      tk('Link|WORD'),
-      tk('||OTHR'),
-      tk('internal|WORD'),
-      tk('link|WORD'),
-      tk(']|OTHR'),
-      tk(']|OTHR'),
-      tk('and|WORD'),
-      tk('[|OTHR'),
-      tk('http://example.com|URLS'),
-      tk('external|WORD'),
-      tk('link|WORD'),
-      tk(']|OTHR'),
-      tk('.|PUNC'),
-      tk('Try|WORD'),
-      tk('_|OTHR'),
-      tk('_|OTHR'),
-      tk('MAGIC|WORD'),
-      tk('_|OTHR'),
-      tk('_|OTHR'),
-      tk('with|WORD'),
-      tk('[|OTHR'),
-      tk('[|OTHR'),
-      tk('Multiline|WORD'),
-      tk('link|WORD'),
-      tk('(|OTHR'),
-      tk('because|WORD'),
-      tk('we|WORD'),
-      tk('can|WORD'),
-      tk(')|OTHR'),
-      tk(']|OTHR'),
-      tk(']|OTHR'),
-      tk('.|PUNC'),
-      tk('[|OTHR'),
-      tk('[|OTHR'),
-      tk('Category|WORD'),
-      tk(':|PUNC'),
-      tk('cat1|WORD'),
-      tk(']|OTHR'),
-      tk(']|OTHR'),
-      tk('Link|WORD'),
-      tk('to|WORD'),
-      tk('[|OTHR'),
-      tk('[|OTHR'),
-      tk(':|PUNC'),
-      tk('Category|WORD'),
-      tk(':|PUNC'),
-      tk('cat2|WORD'),
-      tk(']|OTHR'),
-      tk(']|OTHR'),
-      tk('.|PUNC'),
-      tk('=|OTHR'),
-      tk('=|OTHR'),
-      tk('Heading|WORD'),
-      tk('=|OTHR'),
-      tk('=|OTHR'),
-      tk('{|OTHR'),
-      tk('{|OTHR'),
-      tk('Template|WORD'),
-      tk('}|OTHR'),
-      tk('}|OTHR'),
-      tk('Function|WORD'),
-      tk('with|WORD'),
-      tk('{|OTHR'),
-      tk('{|OTHR'),
-      tk('#|OTHR'),
-      tk('func|WORD'),
-      tk('||OTHR'),
-      tk('param|WORD'),
-      tk('||OTHR'),
-      tk('{|OTHR'),
-      tk('{|OTHR'),
-      tk('{|OTHR'),
-      tk('var|WORD'),
-      tk('}|OTHR'),
-      tk('}|OTHR'),
-      tk('}|OTHR'),
-      tk('}|OTHR'),
-      tk('}|OTHR'),
-      tk('and|WORD'),
-      tk('<|OTHR'),
-      tk('nowiki|WORD'),
-      tk('>|OTHR'),
-      tk('{|OTHR'),
-      tk('{|OTHR'),
-      tk('{|OTHR'),
-      tk('var|WORD'),
-      tk('}|OTHR'),
-      tk('}|OTHR'),
-      tk('}|OTHR'),
-      tk('<|OTHR'),
-      tk('/|OTHR'),
-      tk('nowiki|WORD'),
-      tk('>|OTHR'),
-      tk('!|PUNC'),
-      tk('{|OTHR'),
-      tk('{|OTHR'),
-      tk('Multi|WORD'),
-      tk('||OTHR'),
-      tk('line|WORD'),
-      tk('=|OTHR'),
-      tk('1|NUMS'),
-      tk('||OTHR'),
-      tk('[|OTHR'),
-      tk('[|OTHR'),
-      tk('link|WORD'),
-      tk(']|OTHR'),
-      tk(']|OTHR'),
-      tk('||OTHR'),
-      tk('{|OTHR'),
-      tk('{|OTHR'),
-      tk('{|OTHR'),
-      tk('var|WORD'),
-      tk('}|OTHR'),
-      tk('}|OTHR'),
-      tk('}|OTHR'),
-      tk('}|OTHR'),
-      tk('}|OTHR')
+      tk('Test|WORD|0|0'),
+      tk('[|OTHR|1|5'),
+      tk('[|OTHR|2|6'),
+      tk('Link|WORD|3|7'),
+      tk('||OTHR|4|11'),
+      tk('internal|WORD|5|12'),
+      tk('link|WORD|6|21'),
+      tk(']|OTHR|7|25'),
+      tk(']|OTHR|8|26'),
+      tk('and|WORD|9|28'),
+      tk('[|OTHR|10|32'),
+      tk('http://example.com|URLS|11|33'),
+      tk('external|WORD|12|52'),
+      tk('link|WORD|13|61'),
+      tk(']|OTHR|14|65'),
+      tk('.|PUNC|15|66'),
+      tk('Try|WORD|16|67'),
+      tk('_|OTHR|17|71'),
+      tk('_|OTHR|18|72'),
+      tk('MAGIC|WORD|19|73'),
+      tk('_|OTHR|20|78'),
+      tk('_|OTHR|21|79'),
+      tk('with|WORD|22|81'),
+      tk('[|OTHR|23|86'),
+      tk('[|OTHR|24|87'),
+      tk('Multiline|WORD|25|88'),
+      tk('link|WORD|26|97'),
+      tk('(|OTHR|27|102'),
+      tk('because|WORD|28|103'),
+      tk('we|WORD|29|111'),
+      tk('can|WORD|30|114'),
+      tk(')|OTHR|31|117'),
+      tk(']|OTHR|32|118'),
+      tk(']|OTHR|33|119'),
+      tk('.|PUNC|34|120'),
+      tk('[|OTHR|35|121'),
+      tk('[|OTHR|36|122'),
+      tk('Category|WORD|37|123'),
+      tk(':|PUNC|38|131'),
+      tk('cat1|WORD|39|132'),
+      tk(']|OTHR|40|136'),
+      tk(']|OTHR|41|137'),
+      tk('Link|WORD|42|138'),
+      tk('to|WORD|43|143'),
+      tk('[|OTHR|44|146'),
+      tk('[|OTHR|45|147'),
+      tk(':|PUNC|46|148'),
+      tk('Category|WORD|47|149'),
+      tk(':|PUNC|48|157'),
+      tk('cat2|WORD|49|158'),
+      tk(']|OTHR|50|162'),
+      tk(']|OTHR|51|163'),
+      tk('.|PUNC|52|164'),
+      tk('=|OTHR|53|165'),
+      tk('=|OTHR|54|166'),
+      tk('Heading|WORD|55|168'),
+      tk('=|OTHR|56|176'),
+      tk('=|OTHR|57|177'),
+      tk('{|OTHR|58|178'),
+      tk('{|OTHR|59|179'),
+      tk('Template|WORD|60|180'),
+      tk('}|OTHR|61|188'),
+      tk('}|OTHR|62|189'),
+      tk('Function|WORD|63|190'),
+      tk('with|WORD|64|199'),
+      tk('{|OTHR|65|204'),
+      tk('{|OTHR|66|205'),
+      tk('#|OTHR|67|206'),
+      tk('func|WORD|68|207'),
+      tk('||OTHR|69|211'),
+      tk('param|WORD|70|212'),
+      tk('||OTHR|71|217'),
+      tk('{|OTHR|72|218'),
+      tk('{|OTHR|73|219'),
+      tk('{|OTHR|74|220'),
+      tk('var|WORD|75|221'),
+      tk('}|OTHR|76|224'),
+      tk('}|OTHR|77|225'),
+      tk('}|OTHR|78|226'),
+      tk('}|OTHR|79|227'),
+      tk('}|OTHR|80|228'),
+      tk('and|WORD|81|230'),
+      tk('<|OTHR|82|234'),
+      tk('nowiki|WORD|83|235'),
+      tk('>|OTHR|84|241'),
+      tk('{|OTHR|85|242'),
+      tk('{|OTHR|86|243'),
+      tk('{|OTHR|87|244'),
+      tk('var|WORD|88|245'),
+      tk('}|OTHR|89|248'),
+      tk('}|OTHR|90|249'),
+      tk('}|OTHR|91|250'),
+      tk('<|OTHR|92|251'),
+      tk('/|OTHR|93|252'),
+      tk('nowiki|WORD|94|253'),
+      tk('>|OTHR|95|259'),
+      tk('!|PUNC|96|260'),
+      tk('{|OTHR|97|261'),
+      tk('{|OTHR|98|262'),
+      tk('Multi|WORD|99|263'),
+      tk('||OTHR|100|269'),
+      tk('line|WORD|101|271'),
+      tk('=|OTHR|102|275'),
+      tk('1|NUMS|103|276'),
+      tk('||OTHR|104|278'),
+      tk('[|OTHR|105|280'),
+      tk('[|OTHR|106|281'),
+      tk('link|WORD|107|282'),
+      tk(']|OTHR|108|286'),
+      tk(']|OTHR|109|287'),
+      tk('||OTHR|110|289'),
+      tk('{|OTHR|111|291'),
+      tk('{|OTHR|112|292'),
+      tk('{|OTHR|113|293'),
+      tk('var|WORD|114|294'),
+      tk('}|OTHR|115|297'),
+      tk('}|OTHR|116|298'),
+      tk('}|OTHR|117|299'),
+      tk('}|OTHR|118|300'),
+      tk('}|OTHR|119|301')
     ])
   end
 
   def test_wiki2
     meet({ 'space' => true, 'tags' => true, 'wiki' => true }, @wiki, [
-      tk('Test|WORD'),
-      tk(' |SPAC'),
-      tk('[[|WIKI'),
-      tk('Link|internal link]]|WIKI'),
-      tk(' |SPAC'),
-      tk('and|WORD'),
-      tk(' |SPAC'),
-      tk('[http://|WIKI'),
-      tk('example.com external link]|WIKI'),
-      tk('.|PUNC'),
-      tk('Try|WORD'),
-      tk(' |SPAC'),
-      tk('__MAGIC__|WIKI'),
-      tk(' |SPAC'),
-      tk('with|WORD'),
-      tk(' |SPAC'),
-      tk('[[|WIKI'),
-      tk('Multiline|WIKI'),
-      tk('link (because we can)]]|WIKI'),
-      tk('.|PUNC'),
-      tk('[[|WIKI'),
-      tk('Category:cat1]]|WIKI'),
-      tk('Link|WORD'),
-      tk(' |SPAC'),
-      tk('to|WORD'),
-      tk(' |SPAC'),
-      tk('[[|WIKI'),
-      tk(':Category:cat2]]|WIKI'),
-      tk('.|PUNC'),
-      tk('== Heading ==|WIKI'),
-      tk('{{|WIKI'),
-      tk('Template}}|WIKI'),
-      tk('Function|WORD'),
-      tk(' |SPAC'),
-      tk('with|WORD'),
-      tk(' |SPAC'),
-      tk('{{|WIKI'),
-      tk('#func|param||WIKI'),
-      tk('{{{|WIKI'),
-      tk('var}}}|WIKI'),
-      tk('}}|WIKI'),
-      tk(' |SPAC'),
-      tk('and|WORD'),
-      tk(' |SPAC'),
-      tk('<|HTML'),
-      tk('nowiki>|HTML'),
-      tk('{{{|WIKI'),
-      tk('var}}}|WIKI'),
-      tk('<|HTML'),
-      tk('/nowiki>|HTML'),
-      tk('!|PUNC'),
-      tk('{{|WIKI'),
-      tk('Multi|WIKI'),
-      tk(' | line=1|WIKI'),
-      tk(' | |WIKI'),
-      tk('[[|WIKI'),
-      tk('link]]|WIKI'),
-      tk('|WIKI'),
-      tk(' | |WIKI'),
-      tk('{{{|WIKI'),
-      tk('var}}}|WIKI'),
-      tk('|WIKI'),
-      tk('|WIKI'),
-      tk('}}|WIKI')
+      tk('Test|WORD|0|0'),
+      tk(' |SPAC|1|4'),
+      tk('[[|WIKI|2|5'),
+      tk('Link|internal link]]|WIKI|3|7'),
+      tk(' |SPAC|4|27'),
+      tk('and|WORD|5|28'),
+      tk(' |SPAC|6|31'),
+      tk('[http://|WIKI|7|32'),
+      tk('example.com external link]|WIKI|8|40'),
+      tk('.|PUNC|9|66'),
+      tk('Try|WORD|10|67'),
+      tk(' |SPAC|11|70'),
+      tk('__MAGIC__|WIKI|12|71'),
+      tk(' |SPAC|13|80'),
+      tk('with|WORD|14|81'),
+      tk(' |SPAC|15|85'),
+      tk('[[|WIKI|16|86'),
+      tk('Multiline|WIKI|17|88'),
+      tk('link (because we can)]]|WIKI|18|97'),
+      tk('.|PUNC|19|120'),
+      tk('[[|WIKI|20|121'),
+      tk('Category:cat1]]|WIKI|21|123'),
+      tk('Link|WORD|22|138'),
+      tk(' |SPAC|23|142'),
+      tk('to|WORD|24|143'),
+      tk(' |SPAC|25|145'),
+      tk('[[|WIKI|26|146'),
+      tk(':Category:cat2]]|WIKI|27|148'),
+      tk('.|PUNC|28|164'),
+      tk('== Heading ==|WIKI|29|165'),
+      tk('{{|WIKI|30|178'),
+      tk('Template}}|WIKI|31|180'),
+      tk('Function|WORD|32|190'),
+      tk(' |SPAC|33|198'),
+      tk('with|WORD|34|199'),
+      tk(' |SPAC|35|203'),
+      tk('{{|WIKI|36|204'),
+      tk('#func|param||WIKI|37|206'),
+      tk('{{{|WIKI|38|218'),
+      tk('var}}}|WIKI|39|221'),
+      tk('}}|WIKI|40|227'),
+      tk(' |SPAC|41|229'),
+      tk('and|WORD|42|230'),
+      tk(' |SPAC|43|233'),
+      tk('<|HTML|44|234'),
+      tk('nowiki>|HTML|45|235'),
+      tk('{{{|WIKI|46|242'),
+      tk('var}}}|WIKI|47|245'),
+      tk('<|HTML|48|251'),
+      tk('/nowiki>|HTML|49|252'),
+      tk('!|PUNC|50|260'),
+      tk('{{|WIKI|51|261'),
+      tk('Multi|WIKI|52|263'),
+      tk(' | line=1|WIKI|53|268'),
+      tk(' | |WIKI|54|277'),
+      tk('[[|WIKI|55|280'),
+      tk('link]]|WIKI|56|282'),
+      tk('|WIKI|57|288'),
+      tk(' | |WIKI|58|288'),
+      tk('{{{|WIKI|59|291'),
+      tk('var}}}|WIKI|60|294'),
+      tk('|WIKI|61|300'),
+      tk('|WIKI|62|300'),
+      tk('}}|WIKI|63|300')
+    ])
+  end
+
+  def test_html1
+    meet({ 'tags' => true }, @html, [
+      tk('test|WORD|0|0'),
+      tk('<|HTML|1|5'),
+      tk('a>|HTML|2|6'),
+      tk('test|WORD|3|8'),
+      tk('<|HTML|4|12'),
+      tk('/a>|HTML|5|13'),
+      tk('test|WORD|6|17'),
+      tk('<|HTML|7|21'),
+      tk('b>|HTML|8|22'),
+      tk('test|WORD|9|24'),
+      tk('<|HTML|10|29'),
+      tk('a>|HTML|11|30'),
+      tk('test|WORD|12|32'),
+      tk('<|HTML|13|36'),
+      tk('/a>|HTML|14|37'),
+      tk('<|HTML|15|40'),
+      tk('/b>|HTML|16|41'),
+      tk('test|WORD|17|44'),
+      tk('<|HTML|18|49'),
+      tk('a test="test">|HTML|19|50'),
+      tk('<|HTML|20|64'),
+      tk('b>|HTML|21|65'),
+      tk('test|WORD|22|67'),
+      tk('<|HTML|23|71'),
+      tk('/b>|HTML|24|72'),
+      tk('<|HTML|25|75'),
+      tk('/a>|HTML|26|76'),
+      tk(',|PUNC|27|79'),
+      tk('test|WORD|28|81'),
+      tk('<|HTML|29|85'),
+      tk('a>|HTML|30|86'),
+      tk('test|WORD|31|88'),
+      tk('<|HTML|32|92'),
+      tk('/a>|HTML|33|93'),
+      tk('<|HTML|34|96'),
+      tk('b test="test">|HTML|35|97'),
+      tk('test|WORD|36|111'),
+      tk('<|HTML|37|115'),
+      tk('/b>|HTML|38|116'),
+      tk('<|HTML|39|119'),
+      tk('a>|HTML|40|120'),
+      tk('test|WORD|41|122'),
+      tk('<|HTML|42|126'),
+      tk('/a>|HTML|43|127')
+    ])
+  end
+
+  def test_html2
+    meet({ 'skip-tags' => 'a' }, @html, [
+      tk('test|WORD|0|0'),
+      tk('<|SKIP|1|5'),
+      tk('a>|SKIP|2|6'),
+      tk('test|SKIP|3|8'),
+      tk('<|SKIP|4|12'),
+      tk('/a>|SKIP|5|13'),
+      tk('test|WORD|6|17'),
+      tk('<|HTML|7|21'),
+      tk('b>|HTML|8|22'),
+      tk('test|WORD|9|24'),
+      tk('<|SKIP|10|29'),
+      tk('a>|SKIP|11|30'),
+      tk('test|SKIP|12|32'),
+      tk('<|SKIP|13|36'),
+      tk('/a>|SKIP|14|37'),
+      tk('<|HTML|15|40'),
+      tk('/b>|HTML|16|41'),
+      tk('test|WORD|17|44'),
+      tk('<|SKIP|18|49'),
+      tk('a test="test">|SKIP|19|50'),
+      tk('<|SKIP|20|64'),
+      tk('b>|SKIP|21|65'),
+      tk('test|SKIP|22|67'),
+      tk('<|SKIP|23|71'),
+      tk('/b>|SKIP|24|72'),
+      tk('<|SKIP|25|75'),
+      tk('/a>|SKIP|26|76'),
+      tk(',|PUNC|27|79'),
+      tk('test|WORD|28|81'),
+      tk('<|SKIP|29|85'),
+      tk('a>|SKIP|30|86'),
+      tk('test|SKIP|31|88'),
+      tk('<|SKIP|32|92'),
+      tk('/a>|SKIP|33|93'),
+      tk('<|HTML|34|96'),
+      tk('b test="test">|HTML|35|97'),
+      tk('test|WORD|36|111'),
+      tk('<|HTML|37|115'),
+      tk('/b>|HTML|38|116'),
+      tk('<|SKIP|39|119'),
+      tk('a>|SKIP|40|120'),
+      tk('test|SKIP|41|122'),
+      tk('<|SKIP|42|126'),
+      tk('/a>|SKIP|43|127')
     ])
   end