treat 1.0.6 → 1.1.0

data/lib/treat/config/core/acronyms.rb

@@ -0,0 +1,4 @@
+['xml', 'html', 'txt', 'odt', 
+ 'abw', 'doc', 'yaml', 'uea', 
+ 'lda', 'pdf', 'ptb', 'dot', 
+ 'ai', 'id3', 'svo', 'mlp' ]

data/lib/treat/config/core/encodings.rb

@@ -0,0 +1,8 @@
+{language_to_code: {
+  arabic: 'UTF-8',
+  chinese: 'GB18030',
+  english: 'UTF-8',
+  french: 'ISO_8859-1',
+  ferman: 'ISO_8859-1',
+  hebrew: 'UTF-8'
+}}

data/lib/treat/config/core/entities.rb

@@ -0,0 +1,2 @@
+{list: [:entity, :unknown, :email, :url, :symbol, :sentence, :punctuation, :number, :enclitic, :word, :token, :fragment, :phrase, :paragraph, :title, :zone, :list, :block, :page, :section, :collection, :document],
+order: [:token, :fragment, :phrase, :sentence, :zone, :section, :document, :collection]}

data/lib/treat/config/core/language.rb

@@ -0,0 +1,3 @@
+{default: :english,
+detect: false,
+detect_at: :document}

data/lib/treat/config/core/paths.rb

@@ -0,0 +1,8 @@
+{description: {
+  :tmp => 'temporary files',
+  :lib => 'class and module definitions',
+  :bin => 'binary files',
+  :files => 'user-saved files',
+  :models => 'model files',
+  :spec => 'spec test files'
+}}

data/lib/treat/config/core/syntax.rb

@@ -0,0 +1 @@
+{sweetened: false}

data/lib/treat/config/core/verbosity.rb

@@ -0,0 +1 @@
+{debug: false, silence: true}

data/lib/treat/config/databases/mongo.rb

@@ -0,0 +1,3 @@
+{host: 'localhost',
+ port: '27017',
+ db: nil }

data/lib/treat/config/languages/agnostic.rb

@@ -0,0 +1,34 @@
+{
+  dependencies: [
+    'psych',
+    'nokogiri',
+    'ferret',
+    'bson_ext',
+    'mongo',
+    'lda-ruby',
+    'stanford-core-nlp',
+    'linguistics',
+    'ruby-readability',
+    'whatlanguage',
+    'chronic',
+    'nickel',
+    'decisiontree',
+    'ai4r'
+  ],
+  workers: {
+    extractors: {
+      keywords: [:tf_idf],
+      language: [:what_language]
+    },
+    formatters: {
+      serializers: [:xml, :yaml, :mongo]
+    },
+    lexicalizers: {
+      categorizers: [:from_tag]
+    },
+    inflectors: {
+      ordinalizers: [:linguistics],
+      cardinalizers: [:linguistics]
+    }
+  }
+}

data/lib/treat/config/languages/arabic.rb

@@ -0,0 +1,13 @@
+{
+  dependencies: [
+    'stanford-core-nlp'
+  ],
+  workers: {
+    processors: {
+      parsers: [:stanford]
+    },
+    lexicalizers: {
+      taggers: [:stanford]
+    }
+  }
+}

data/lib/treat/config/languages/chinese.rb

@@ -0,0 +1,13 @@
+{
+  dependencies: [
+    'stanford-core-nlp'
+  ],
+  workers: {
+    processors: {
+      parsers: [:stanford]
+    },
+    lexicalizers: {
+      taggers: [:stanford]
+    }
+  }
+}

data/lib/treat/config/languages/dutch.rb

@@ -0,0 +1,12 @@
+{
+  dependencies: [
+    'punkt-segmenter', 
+    'tactful_tokenizer'
+  ],
+  workers: {
+    processors: {
+      segmenters: [:punkt],
+      tokenizers: [:tactful]
+    }
+  }
+}

data/lib/treat/config/languages/english.rb

@@ -0,0 +1,60 @@
+{
+  dependencies: [
+    'rbtagger', 
+    'ruby-stemmer', 
+    'punkt-segmenter', 
+    'tactful_tokenizer', 
+    'nickel', 
+    'rwordnet', 
+    'uea-stemmer', 
+    'engtagger', 
+    'activesupport', 
+    'english'
+  ],
+  workers: {
+    extractors: {
+      time: [:chronic, :ruby, :nickel],
+      topics: [:reuters],
+      keywords: [:tf_idf],
+      name_tag: [:stanford],
+      coreferences: [:stanford]
+    },
+    inflectors: {
+      conjugators: [:linguistics],
+      declensors: [:english, :linguistics, :active_support],
+      stemmers: [:porter, :porter_c, :uea],
+      ordinalizers:  [:linguistics],
+      cardinalizers:  [:linguistics]
+    },
+    lexicalizers: {
+      taggers: [:lingua, :brill, :stanford],
+      sensers: [:wordnet]
+    },
+    processors: {
+      parsers: [:stanford, :enju],
+      segmenters: [:tactful, :punkt, :stanford],
+      tokenizers: [:ptb, :stanford, :tactful, :punkt]
+    }
+  },
+  info: {
+    stopwords:
+      ['the', 'of', 'and', 'a', 'to', 'in', 'is',
+      'you', 'that', 'it', 'he', 'was', 'for', 'on',
+      'are', 'as', 'with', 'his', 'they', 'I', 'at',
+      'be', 'this', 'have', 'from', 'or', 'one', 'had',
+      'by', 'word', 'but', 'not', 'what', 'all', 'were',
+      'we', 'when', 'your', 'can', 'said', 'there', 'use',
+      'an', 'each', 'which', 'she', 'do', 'how', 'their',
+      'if', 'will', 'up', 'other', 'about', 'out', 'many',
+      'then', 'them', 'these', 'so', 'some', 'her', 'would',
+      'make', 'like', 'him', 'into', 'time', 'has', 'look',
+      'two', 'more', 'write', 'go', 'see', 'number', 'no',
+      'way', 'could', 'people', 'my', 'than', 'first', 'been',
+      'call', 'who', 'its', 'now', 'find', 'long', 'down',
+      'day', 'did', 'get', 'come', 'made', 'may', 'part',
+      'say', 'also', 'new', 'much', 'should', 'still',
+      'such', 'before', 'after', 'other', 'then', 'over',
+      'under', 'therefore', 'nonetheless', 'thereafter',
+      'afterwards', 'here', 'huh', 'hah', "n't", "'t", 'here']
+  }
+}

data/lib/treat/config/languages/french.rb

@@ -0,0 +1,18 @@
+{
+  dependencies: [
+    'punkt-segmenter', 
+    'tactful_tokenizer', 
+    'stanford-core-nlp'
+  ],
+  workers: {
+    processors: {
+      segmenters: [:punkt],
+      tokenizers: [:tactful],
+      parsers: [:stanford]
+    },
+    lexicalizers: {
+      taggers: [:stanford],
+      categorizers: [:from_tag]
+    }
+  }
+}

data/lib/treat/config/languages/german.rb

@@ -0,0 +1,18 @@
+{
+  dependencies: [
+    'punkt-segmenter', 
+    'tactful_tokenizer', 
+    'stanford'
+  ],
+  workers: {
+    processors: {
+      segmenters: [:punkt],
+      tokenizers: [:tactful],
+      parsers: [:stanford]
+    },
+    lexicalizers: {
+      taggers: [:stanford],
+      categorizers: [:from_tag]
+    }
+  }
+}

data/lib/treat/config/languages/greek.rb

@@ -0,0 +1,12 @@
+{
+  dependencies: [
+    'punkt-segmenter', 
+    'tactful_tokenizer'
+  ],
+  workers: {
+    processors: {
+      segmenters: [:punkt],
+      tokenizers: [:tactful]
+    }
+  }
+}

data/lib/treat/config/languages/italian.rb

@@ -0,0 +1,12 @@
+{
+  dependencies: [
+    'punkt-segmenter', 
+    'tactful_tokenizer'
+  ],
+  workers: {
+    processors: {
+      segmenters: [:punkt],
+      tokenizers: [:tactful]
+    }
+  }
+}

data/lib/treat/config/languages/polish.rb

@@ -0,0 +1,12 @@
+{
+  dependencies: [
+    'punkt-segmenter', 
+    'tactful_tokenizer'
+  ],
+  workers: {
+    processors: {
+      segmenters: [:punkt],
+      tokenizers: [:tactful]
+    }
+  }
+}

data/lib/treat/config/languages/portuguese.rb

@@ -0,0 +1,12 @@
+{
+  dependencies: [
+    'punkt-segmenter', 
+    'tactful_tokenizer'
+  ],
+  workers: {
+    processors: {
+      segmenters: [:punkt],
+      tokenizers: [:tactful]
+    }
+  }
+}

data/lib/treat/config/languages/russian.rb

@@ -0,0 +1,12 @@
+{
+  dependencies: [
+    'punkt-segmenter', 
+    'tactful_tokenizer'
+  ],
+  workers: {
+    processors: {
+      segmenters: [:punkt],
+      tokenizers: [:tactful]
+    }
+  }
+}

data/lib/treat/config/languages/spanish.rb

@@ -0,0 +1,12 @@
+{
+  dependencies: [
+    'punkt-segmenter', 
+    'tactful_tokenizer'
+  ],
+  workers: {
+    processors: {
+      segmenters: [:punkt],
+      tokenizers: [:tactful]
+    }
+  }
+}

data/lib/treat/config/languages/swedish.rb

@@ -0,0 +1,12 @@
+{
+  dependencies: [
+    'punkt-segmenter', 
+    'tactful_tokenizer'
+  ],
+  workers: {
+    processors: {
+      segmenters: [:punkt],
+      tokenizers: [:tactful]
+    }
+  }
+}

data/lib/treat/config/libraries/stanford.rb

@@ -0,0 +1 @@
+{jar_path: nil, model_path: nil}

data/lib/treat/config/linguistics/categories.rb

@@ -0,0 +1,4 @@
+['adjective', 'adverb', 'noun', 'verb', 'interjection',
+ 'clitic', 'coverb', 'conjunction', 'determiner', 'particle',
+ 'preposition', 'pronoun', 'number', 'symbol', 'punctuation',
+ 'complementizer']

data/lib/treat/config/linguistics/punctuation.rb

@@ -0,0 +1,33 @@
+{punct_to_category: {
+  '.' => 'period',
+  ',' => 'comma',
+  ';' => 'semicolon',
+  ':' => 'colon',
+  '?' => 'interrogation',
+  '!' => 'exclamation',
+  '"' => 'double_quote',
+  "'" => 'single_quote',
+  '$' => 'dollar',
+  '%' => 'percent',
+  '#' => 'hash',
+  '*' => 'asterisk',
+  '&' => 'ampersand',
+  '+' => 'plus',
+  '-' => 'dash',
+  '/' => 'slash',
+  '\\' => 'backslash',
+  '^' => 'caret',
+  '_' => 'underscore',
+  '`' => 'tick',
+  '|' => 'pipe',
+  '~' => 'tilde',
+  '@' => 'at',
+  '[' => 'bracket',
+  ']' => 'bracket',
+  '{' => 'brace',
+  '}' => 'brace',
+  '(' => 'parenthesis',
+  ')' => 'parenthesis',
+  '<' => 'tag',
+  '>' => 'tag'
+}}

data/lib/treat/config/tags/aligned.rb

@@ -0,0 +1,221 @@
+{tag_sets: [
+  :claws_c5, :brown, :penn, :stutgart, :chinese, :paris7
+],
+phrase_tags: [
+  'Adjectival phrase', ['', '', 'ADJP', '', '', 'AP'],
+  'Adverbial phrase', ['', '', 'ADVP', '', '', 'AdP'],
+  'Conjunction phrase', ['', '', 'CONJP', '', '', 'Ssub'],
+  'Fragment', ['', '', 'FRAG', '', '', ''],
+  'Interjectional phrase', ['', '', 'INTJ', '', '', ''],
+  'List marker', ['', '', 'LST', '', '', ''],
+  'Not a phrase', ['', '', 'NAC', '', '', ''],
+  'Noun phrase', ['', '', 'NP', '', '', 'NP'],
+  'Verbal nucleus',  ['', '', '', '', '', 'VN'],
+  'Head of noun phrase', ['', '', 'NX', '', '', ''],
+  'Prepositional phrase', ['', '', 'PP', '', '', 'PP'],
+  'Parenthetical', ['', '', 'PRN', '', '', ''],
+  'Particle', ['', '', 'PRT', '', '', ''],
+  'Participial phrase', ['', '', '', '', '', 'VPart'],
+  'Quantifier phrase', ['', '', 'QP', '', '', ''],
+  'Relative clause', ['', '', 'RRC', '', '', 'Srel'],
+  'Coordinated phrase', ['', '', 'UCP', '', '', 'COORD'],
+  'Infinitival phrase', ['', '', '', '', '', 'VPinf'], 
+  'Verb phrase', ['', '', 'VP', '', '', ''],
+  'Wh adjective phrase', ['', '', 'WHADJP', '', '', ''],
+  'Wh adverb phrase', ['', '', 'WHAVP', '', '', ''],
+  'Wh noun phrase', ['', '', 'WHNP', '', '', ''],
+  'Wh prepositional phrase', ['', '', 'WHPP', '', '', ''],
+  'Unknown', ['', '', 'X', '', '', ''],
+  'Phrase', ['', '', 'P', '', '', 'Sint'],
+  'Sentence', ['', '', 'S', '', '', 'SENT'],
+  'Phrase', ['', '', 'SBAR', '', '', ''] # Fix
+],
+word_tags: [
+  
+  # Aligned tags for the Claws C5, Brown and Penn tag sets.
+  # Adapted from Manning, Christopher and Schütze, Hinrich,
+  # 1999. Foundations of Statistical Natural Language
+  # Processing. MIT Press, p. 141-142;
+  # http://www.isocat.org/rest/dcs/376;
+
+  'Adjective', ['AJ0', 'JJ', 'JJ', '', 'JJ', 'A'],
+  'Adjective', ['AJ0', 'JJ', 'JJ', '', 'JJ', 'ADJ'],
+  'Ajective, adverbial or predicative', ['', '', '', 'ADJD', '', 'ADJ'],
+  'Adjective, attribute', ['', '', '', 'ADJA', 'VA', 'ADJ'],
+  'Adjective, ordinal number', ['ORD', 'OD', 'JJ', '', 'OD', 'ADJ'],
+  'Adjective, comparative', ['AJC', 'JJR', 'JJR', 'KOKOM', '', 'ADJ'],
+  'Adjective, superlative', ['AJS', 'JJT', 'JJS', '', 'JJ', 'ADJ'],
+  'Adjective, superlative, semantically', ['AJ0', 'JJS', 'JJ', '', '', 'ADJ'],
+  'Adjective, cardinal number', ['CRD', 'CD', 'CD', 'CARD', 'CD', 'ADJ'],
+  'Adjective, cardinal number, one', ['PNI', 'CD', 'CD', 'CARD', 'CD', 'ADJ'],
+
+  'Adverb', ['AV0', 'RB', 'RB', 'ADV', 'AD', 'ADV'],
+  'Adverb, negative', ['XX0', '*', 'RB', 'PTKNEG', '', 'ADV'],
+  'Adverb, comparative', ['AV0', 'RBR', 'RBR', '', 'AD', 'ADV'],
+  'Adverb, superlative', ['AV0', 'RBT', 'RBS', '', 'AD', 'ADV'],
+  'Adverb, particle', ['AVP', 'RP', 'RP', '', '', 'ADV'],
+  'Adverb, question', ['AVQ', 'WRB', 'WRB', '', 'AD', 'ADV'],
+  'Adverb, degree & question', ['AVQ', 'WQL', 'WRB', '', 'ADV'],
+  'Adverb, degree', ['AV0', 'QL', 'RB', '', '', 'ADV'],
+  'Adverb, degree, postposed', ['AV0', 'QLP', 'RB', '', '', 'ADV'],
+  'Adverb, nominal', ['AV0', 'RN', 'RB', 'PROP', '', 'ADV'],
+  'Adverb, pronominal', ['', '', '', '', 'PROP', '', 'ADV'],
+
+  'Conjunction, coordination', ['CJC', 'CC', 'CC', 'KON', 'CC', 'COOD'],
+  'Conjunction, coordination, and', ['CJC', 'CC', 'CC', 'KON', 'CC', 'ET'],
+  'Conjunction, subordination', ['CJS', 'CS', 'IN', 'KOUS', 'CS', 'CONJ'],
+  'Conjunction, subordination with to and infinitive', ['', '', '', 'KOUI', '', ''],
+  'Conjunction, complementizer, that', ['CJT', 'CS', 'IN', '', '', 'C'],
+
+  'Determiner', ['DT0', 'DT', 'DT', '', 'DT', 'D'],
+  'Determiner, pronoun', ['DT0', 'DTI', 'DT', '', '', 'D'],
+  'Determiner, pronoun, plural', ['DT0', 'DTS', 'DT', '', '', 'D'],
+  'Determiner, prequalifier', ['DT0', 'ABL', 'DT', '', '', 'D'],
+  'Determiner, prequantifier', ['DT0', 'ABN', 'PDT', '', 'DT', 'D'],
+  'Determiner, pronoun or double conjunction', ['DT0', 'ABX', 'PDT', '', '', 'D'],
+  'Determiner, pronoun or double conjunction', ['DT0', 'DTX', 'DT', '', '', 'D'],
+  'Determiner, article', ['AT0', 'AT', 'DT', 'ART', '', 'D'],
+  'Determiner, postdeterminer', ['DT0', 'AP', 'DT', '', '', 'D'],
+  'Determiner, possessive', ['DPS', 'PP$', 'PRP$', '', '', 'D'],
+  'Determiner, possessive, second', ['DPS', 'PP$', 'PRPS', '', '', 'D'],
+  'Determiner, question', ['DTQ', 'WDT', 'WDT', '', 'DT', 'D'],
+  'Determiner, possessive & question', ['DTQ', 'WP$', 'WP$', '', '', 'D'],
+  'Interjection', ['', '', '', '', '', 'I'],
+  'Localizer', ['', '', '', '', 'LC'],
+
+  'Measure word', ['', '', '', '', 'M'],
+
+  'Noun, common', ['NN0', 'NN', 'NN', 'N', 'NN', 'NN'],
+  'Noun, singular', ['NN1', 'NN', 'NN', 'NN', 'NN', 'N'],
+  'Noun, plural', ['NN2', 'NNS', 'NNS', 'NN', 'NN', 'N'],
+  'Noun, proper, singular', ['NP0', 'NP', 'NNP', 'NE', 'NR', 'N'],
+  'Noun, proper, plural', ['NP0', 'NPS', 'NNPS', 'NE', 'NR', 'N'],
+  'Noun, adverbial', ['NN0', 'NR', 'NN', 'NE', '', 'N'],
+  'Noun, adverbial, plural', ['NN2', 'NRS', 'NNS', '', 'N'],
+  'Noun, temporal', ['', '', '', '', 'NT', 'N'],
+  'Noun, verbal', ['', '', '', '', 'NN', 'N'],
+
+  'Pronoun, nominal (indefinite)', ['PNI', 'PN', 'PRP', '', 'PN', 'CL'],
+  'Pronoun, personal, subject', ['PNP', 'PPSS', 'PRP', 'PPER'],
+  'Pronoun, personal, subject, 3SG', ['PNP', 'PPS', 'PRP', 'PPER'],
+  'Pronoun, personal, object', ['PNP', 'PPO', 'PRP', 'PPER'],
+  'Pronoun, reflexive', ['PNX', 'PPL', 'PRP', 'PRF'],
+  'Pronoun, reflexive, plural', ['PNX', 'PPLS', 'PRP', 'PRF'],
+  'Pronoun, question, subject', ['PNQ', 'WPS', 'WP', 'PWAV'],
+  'Pronoun, question, subject', ['PNQ', 'WPS', 'WPS', 'PWAV'],  # Hack
+  'Pronoun, question, object', ['PNQ', 'WPO', 'WP', 'PWAV', 'PWAT'],
+  'Pronoun, existential there', ['EX0', 'EX', 'EX'],
+  'Pronoun, attributive demonstrative', ['', '', '', 'PDAT'],
+  'Prounoun, attributive indefinite without determiner', ['', '', '', 'PIAT'],
+  'Pronoun, attributive possessive', ['', '', '', 'PPOSAT', ''],
+  'Pronoun, substituting demonstrative', ['', '', '', 'PDS'],
+  'Pronoun, substituting possessive', ['', '', '', 'PPOSS', ''],
+  'Prounoun, substituting indefinite', ['', '', '', 'PIS'],
+  'Pronoun, attributive relative', ['', '', '', 'PRELAT', ''],
+  'Pronoun, substituting relative', ['', '', '', 'PRELS', ''],
+  'Pronoun, attributive interrogative', ['', '', '', 'PWAT'],
+  'Pronoun, adverbial interrogative', ['', '', '', 'PWAV'],
+
+  'Pronoun, substituting interrogative', ['', '', '', 'PWS'],
+  'Verb, main, finite', ['', '', '', 'VVFIN', '', 'V'],
+  'Verb, main, infinitive', ['', '', '', 'VVINF', '', 'V'],
+  'Verb, main, imperative', ['', '', '', 'VVIMP', '', 'V'],
+  'Verb, base present form (not infinitive)', ['VVB', 'VB', 'VBP', '', '', 'V'],
+  'Verb, infinitive', ['VVI', 'VB', 'VB', 'V', '', 'V'],
+  'Verb, past tense', ['VVD', 'VBD', 'VBD', '', '', 'V'],
+  'Verb, present participle', ['VVG', 'VBG', 'VBG', 'VAPP', '', 'V'],
+  'Verb, past/passive participle', ['VVN', 'VBN', 'VBN', 'VVPP', '', 'V'],
+  'Verb, present, 3SG, -s form', ['VVZ', 'VBZ', 'VBZ', '', '', 'V'],
+  'Verb, auxiliary', ['', '', '', 'VAFIN', '', 'V'],
+  'Verb, imperative', ['', '', '', 'VAIMP', '', 'V'],
+  'Verb, imperative infinitive', ['', '', '', 'VAINF', '', 'V'],
+  'Verb, auxiliary do, base', ['VDB', 'DO', 'VBP', '', '', 'V'],
+  'Verb, auxiliary do, infinitive', ['VDB', 'DO', 'VB', '', '', 'V'],
+  'Verb, auxiliary do, past', ['VDD', 'DOD', 'VBD', '', '', 'V'],
+  'Verb, auxiliary do, present participle', ['VDG', 'VBG', 'VBG', '', '', 'V'],
+  'Verb, auxiliary do, past participle', ['VDN', 'VBN', 'VBN', '', '', 'V'],
+  'Verb, auxiliary do, present 3SG', ['VDZ', 'DOZ', 'VBZ', '', '', 'V'],
+  'Verb, auxiliary have, base', ['VHB', 'HV', 'VBP', 'VA', '', 'V'],
+  'Verb, auxiliary have, infinitive', ['VHI', 'HV', 'VB', 'VAINF', '', 'V'],
+  'Verb, auxiliary have, past', ['VHD', 'HVD', 'VBD', 'VA', '', 'V'],
+  'Verb, auxiliary have, present participle', ['VHG', 'HVG', 'VBG', 'VA', '', 'V'],
+  'Verb, auxiliary have, past participle', ['VHN', 'HVN', 'VBN', 'VAPP', '', 'V'],
+  'Verb, auxiliary have, present 3SG', ['VHZ', 'HVZ', 'VBZ', 'VA', '', 'V'],
+  'Verb, auxiliary be, infinitive', ['VBI', 'BE', 'VB', '', '', 'V'],
+  'Verb, auxiliary be, past', ['VBD', 'BED', 'VBD', '', '', 'V'],
+  'Verb, auxiliary be, past, 3SG', ['VBD', 'BEDZ', 'VBD', '', '', 'V'],
+  'Verb, auxiliary be, present participle', ['VBG', 'BEG', 'VBG', '', '', 'V'],
+  'Verb, auxiliary be, past participle', ['VBN', 'BEN', 'VBN', '', '', 'V'],
+  'Verb, auxiliary be, present, 3SG', ['VBZ', 'BEZ', 'VBZ', '', '', 'V'],
+  'Verb, auxiliary be, present, 1SG', ['VBB', 'BEM', 'VBP', '', '', 'V'],
+  'Verb, auxiliary be, present', ['VBB', 'BER', 'VBP', '', '', 'V'],
+  'Verb, modal', ['VM0', 'MD', 'MD', 'VMFIN', 'VV', 'V'],
+  'Verb, modal', ['VM0', 'MD', 'MD', 'VMINF', 'VV', 'V'],
+  'Verb, modal, finite', ['', '', '', '', 'VMFIN', 'V'],
+  'Verb, modal, infinite', ['', '', '', '', 'VMINF', 'V'],
+  'Verb, modal, past participle', ['', '', '', '', 'VMPP', 'V'],
+
+  'Particle', ['', '', '', '', '', 'PRT'],
+  'Particle, with adverb', ['', '', '', 'PTKA', '', 'PRT'],
+  'Particle, answer', ['', '', '', 'PTKANT', '', 'PRT'],
+  'Particle, negation', ['', '', '', 'PTKNEG', '', 'PRT'],
+  'Particle, separated verb', ['', '', '', 'PTKVZ', '', 'PRT'],
+  'Particle, to as infinitive marker', ['TO0', 'TO', 'TO', 'PTKZU', '', 'PRT'],
+
+  'Preposition, comparative', ['', '', '', 'KOKOM', '', 'P'],
+  'Preposition, to', ['PRP', 'IN', 'TO', '', '', 'P'],
+  'Preposition', ['PRP', 'IN', 'IN', 'APPR', 'P', 'P'],
+  'Preposition, with aritcle', ['', '', '', 'APPART', '', 'P'],
+  'Preposition, of', ['PRF', 'IN', 'IN', '', '', 'P'],
+
+  'Possessive', ['POS', '$', 'POS'],
+
+  'Postposition', ['', '', '', 'APPO'],
+
+  'Circumposition, right', ['', '', '', 'APZR', ''],
+
+  'Interjection, onomatopoeia or other isolate', ['ITJ', 'UH', 'UH', 'ITJ', 'IJ'],
+
+  'Onomatopoeia', ['', '', '', '', 'ON'],
+
+  'Punctuation', ['', '', '', '', 'PU', 'PN'],
+  'Punctuation, sentence ender', ['PUN', '.', '.', '', '', 'PN'],
+
+  'Punctuation, semicolon', ['PUN', '.', '.', '', '', 'PN'],
+  'Puncutation, colon or ellipsis', ['PUN', ':', ':'],
+  'Punctuationm, comma', ['PUN', ',', ',', '$,'],
+  'Punctuation, dash', ['PUN', '-', '-'],
+  'Punctuation, dollar sign', ['PUN', '', '$'],
+  'Punctuation, left bracket', ['PUL', '(', '(', '$('],
+  'Punctuation, right bracket', ['PUR', ')', ')'],
+  'Punctuation, quotation mark, left', ['PUQ', '', '``'],
+  'Punctuation, quotation mark, right', ['PUQ', '', '"'],
+
+  'Punctuation, left bracket', ['PUL', '(', 'PPL'],
+  'Punctuation, right bracket', ['PUR', ')', 'PPR'],
+  'Punctuation, left square bracket', ['PUL', '(', 'LSB'],
+  'Punctuation, right square bracket', ['PUR', ')', 'RSB'],
+  'Punctuation, left curly bracket', ['PUL', '(', 'LCB'],
+  'Punctuation, right curly bracket', ['PUR', ')', 'RCB'],
+
+  'Unknown, foreign words (not in lexicon)', ['UNZ', '(FW-)', 'FW', '', 'FW'],
+
+  'Symbol', ['', '', 'SYM', 'XY'],
+  'Symbol, alphabetical', ['ZZ0', '', ''],
+  'Symbol, list item', ['', '', 'LS'],
+
+  # Not sure about these tags from the Chinese PTB.
+  'Aspect marker', ['', '', '', '', 'AS'],                         # ?
+  'Ba-construction', ['', '', '', '', 'BA'],                       # ?
+  'In relative', ['', '', '', '', 'DEC'],                          # ?
+  'Associative', ['', '', '', '', 'DER'],                          # ?
+  'In V-de or V-de-R construct', ['', '', '', '', 'DER'],          # ?
+  'For words ? ', ['', '', '', '', 'ETC'],                         # ?
+  'In long bei-construct', ['', '', '', '', 'LB'],                 # ?
+  'In short bei-construct', ['', '', '', '', 'SB'],                # ?
+  'Sentence-nal particle', ['', '', '', '', 'SB'],                 # ?
+  'Particle, other', ['', '', '', '', 'MSP'],                      # ?
+  'Before VP', ['', '', '', '', 'DEV'],                            # ?
+  'Verb, ? as main verb', ['', '', '', '', 'VE'],                  # ?
+  'Verb, ????', ['', '', '', '', 'VC']                             # ?
+]}