proiel-cli 0.1.0

data/lib/proiel/cli/converters/conll-u/morphology.rb

@@ -0,0 +1,235 @@
+# -*- coding: utf-8 -*-
+module PROIEL
+  module Converter
+    class CoNLLU
+      COPULAR_LEMMATA = ['sum,V-,lat', 'εἰμί#1,V-,grc']
+      DETERMINERS = ['S-', 'Pd', 'Px', 'Ps', 'Pt']
+      NEGATION_LEMMATA = ['non,Df,lat', 'ne,Df,lat', 
+                          'μή,Df,grc',
+                          'μήγε,Df,grc',
+                          'μηδαμῶς,Df,grc',
+                          'μηδέποτε,Df,grc',
+                          'μηδέπω,Df,grc',
+                          'μηκέτι,Df,grc',
+                          'μήπω,Df,grc',
+                          'μήτε,Df,grc',
+                          'μήτι,Df,grc',
+                          'μήτιγε,Df,grc',
+                          'οὐ,Df,grc',
+                          'οὐδαμῇ,Df,grc',
+                          'οὐδαμῶς,Df,grc',
+                          'οὐδέ,Df,grc',
+                          'οὐδέποτε,Df,grc',
+                          'οὐδέπω,Df,grc',
+                          'οὐκέτι,Df,grc',
+                          'οὐκοῦν,Df,grc',
+                          'οὔπω,Df,grc',
+                          'οὔτε,Df,grc',
+                          'οὔτι,Df,grc',
+                          'οὐχί,Df,grc',
+                          'не,Df,chu',
+                          'ни,Df,chu',
+                          'нѣ,Df,chu',
+                          'nei,Df,got',
+                          'ni,Df,got',
+                          'nibai#2,Df,got',
+                          'nih,Df,got',
+                         ]
+
+
+      PARTICLE_LEMMATA = [ 'at,Df,lat',
+                           'atque,Df,lat',
+                           'autem,Df,lat',
+                           'certe,Df,lat',
+                           'ergo,Df,lat',
+                           'et,Df,lat',
+                           'enim,Df,lat',
+                           'etiam,Df,lat',
+                           'igitur,Df,lat',
+                           'immo,Df,lat',
+                           'itaque,Df,lat',
+                           'nam,Df,lat',
+                           'nonne,Df,lat',
+                           'nonne,Du,lat',
+                           'quidem,Df,lat',
+                           'quoque,Df,lat',
+                           'sic,Df,lat',
+                           'tamen,Df,lat',
+                           'tum,Df,lat',
+                           'tunc,Df,lat',
+                           'vero,Df,lat',
+                           'ἅμα,Df,grc',
+                           'ἄν,Df,grc',
+                           'ἀνά,Df,grc',
+                           'ἆρα,Df,grc',
+                           'ἄραγε,Df,grc',
+                           'ἀτάρ,Df,grc',
+                           'ἅτε,Df,grc',
+                           'αὗ,Df,grc',
+                           'αὖθις,Df,grc',
+                           'γάρ,Df,grc',
+                           'γε,Df,grc',
+                           'γοῦν,Df,grc',
+                           'δέ,Df,grc',
+                           'δή,Df,grc',
+                           'δῆθεν,Df,grc',
+                           'δηλαδή,Df,grc',
+                           'δηλονότι,Df,grc',
+                           'δῆτα,Df,grc',
+                           'εἶτα,Df,grc',
+                           'ἔτι,Df,grc',
+                           'ἦ#2,Df,grc',
+                           'ἤγουν,Df,grc',
+                           'ἤδη,Df,grc',
+                           'ἤτοι,Df,grc',
+                           'καίτοι,Df,grc',
+                           'καίτοιγε,Df,grc',
+                           'μέν,Df,grc',
+                           'μενοῦνγε,Df,grc',
+                           'μέντοι,Df,grc',
+                           'μήν,Df,grc',
+                           'νά,Df,grc',
+                           'νῦν#1,Df,grc',
+                           'νυν#2,Df,grc',
+                           'νυνί,Df,grc',
+                           'οὖν,Df,grc',
+                           'πέρ,Df,grc',
+                           'πῃ,Df,grc',
+                           'ποτε,Df,grc',
+                           'πού,Df,grc',
+                           'πω,Df,grc',
+                           'πως,Df,grc',
+                           'τάχα,Df,grc',
+                           'τε,Df,grc',
+                           'τοι,Df,grc',
+                           'τοιγαροῦν,Df,grc',
+                           'τοίνυν,Df,grc',
+                           'бо,Df,chu',
+                           'же,Df,chu',
+                           'занѥ,Df,chu',
+                           'ибо,Df,chu',
+                           'иде,Df,chu',
+                           'ижде,Df,chu',
+                           'ли,Df,chu',
+                           'обаче,Df,chu',
+                           'оубо,Df,chu',
+                           'ти,Df,chu',
+                           'тѣ,Df,chu',
+                           'ꙗко#2,Df,chu',
+                           'an,Df,got',
+                           'auk,Df,got',
+                           'aufto,Df,got',
+                           'nu,Df,got',
+                           'ussindo,Df,got',
+                           'waitei,Df,got',
+                           'þan,Df,got',
+                           'nuh,Df,got',
+                           'nunu,Df,got',
+                           'raihtis,Df,got',
+                           'sunsaiw,Df,got',
+                           'unte,Df,got',
+                           'þande,Df,got',
+                           'þannu,Df,got',
+                           'þanuh,Df,got',
+                           'þaruh,Df,got',
+                         ]
+
+      
+      POS_MAP = 
+        { 
+          'A-' => ['ADJ'],
+          'Df' => ['ADV'],
+          'S-' => ['DET', "Definite=Def|PronType=Dem"], # (we only have definite articles)
+          'Ma' => ['NUM'], 
+          'Nb' => ['NOUN'],
+          'C-' => ['CONJ'],
+          'Pd' => ['DET'], 
+          'F-' => ['X'],
+          'Px' => ['PRON'], 
+          'N-' => ['SCONJ'], #irrelevant for our purposes
+          'I-' => ['INTJ'],
+          'Du' => ['ADV', "PronType=Int"],
+          'Pi' => ['PRON', "PronType=Int"],
+          'Mo' => ['ADJ'], 
+          'Pp' => ['PRON', "PronType=Prs"],
+          'Pk' => ['PRON', "PronType=Prs|Reflex=Yes"],
+          'Ps' => ['PRON', "PronType=Prs|Poss=Yes"],   ###  layered gender?
+          'Pt' => ['PRON', "PronType=Prs|Poss=Yes|Reflex=Yes" ],   ###  layered gender? 
+          'R-' => ['ADP'],
+          'Ne' => ['PROPN'],
+          'Py' => ['DET'], 
+          'Pc' => ['PRON', "PronType=Rcp"],
+          'Dq' => ['ADV', "PronType=Rel"],
+          'Pr' => ['PRON', "PronType=Rel"],
+          'G-' => ['SCONJ'],
+          'V-' => ['VERB'],
+          'X-' => ['X'] }
+      
+      MORPHOLOGY_MAP = {
+        :person => {'1' => 'Person=1', 
+                    '2' => 'Person=2', 
+                    '3' => 'Person=3'  } , 
+        :number => {'s' => 'Number=Sing', 
+                    'd' => 'Number=Dual', 
+                    'p' => 'Number=Plur'  } ,
+        :tense  => {'p' => 'Tense=Pres', 
+                    'i' => 'Tense=Past|Aspect=Imp', 
+                    'r' => 'Tense=Past|Aspect=Perf', #'Tense=Perfect', 
+                    's' => 'Aspect=Res',
+                    # tags Perf is not universal
+                    'a' => 'Tense=Past|Aspect=Perf', 
+                    'u' => 'Tense=Past', 
+                    'l' => 'Tense=Pqp', 
+                    'f' => 'Tense=Fut', 
+                    # tag FutPerfect is not universal
+                    't' => 'Tense=Fut|Aspect=Perf', #FutPerfect' 
+                      },
+        :mood =>   {'i' => 'VerbForm=Fin|Mood=Ind', 
+                    's' => 'VerbForm=Fin|Mood=Sub', 
+                    'm' => 'VerbForm=Fin|Mood=Imp', 
+                    'o' => 'VerbForm=Fin|Mood=Opt', 
+                    'n' => 'VerbForm=Inf', 
+                    'p' => 'VerbForm=Part', 
+                    'd' => 'VerbForm=Ger', 
+                    # Gdv (gerundive) is not universal
+                    'g' => 'VerbForm=Gdv', 
+                    'u' => 'VerbForm=Sup', 
+                    'e'=> 'VerbForm=Fin|Mood=Ind,Sub', 
+                    'f'=> 'VerbForm=Fin|Mood=Imp,Ind', 
+                    'h'=> 'VerbForm=Fin|Mood=Imp,Sub', 
+                    't' => 'VerbForm=Fin' },
+        :voice =>  {'a' => 'Voice=Act', 
+                    # Med is not universal
+                    'm' => 'Voice=Mid', 
+                    'p' => 'Voice=Pass', 
+                    'e' => 'Voice=Mid,Pass' },
+        :gender => {'m' => 'Gender=Masc',
+                    'f' => 'Gender=Fem',
+                    'n' => 'Gender=Neut',
+                    'p' => 'Gender=Fem,Masc',
+                    'o' => 'Gender=Masc,Neut',
+                    'r' => 'Gender=Fem,Neut' },
+        :case =>   {'n' => 'Case=Nom', 
+                    'a' => 'Case=Acc', 
+                    # Obl(ique) is not universal
+                    'o' => 'Case=Obl', 
+                    'g' => 'Case=Gen', 
+                    'c' => 'Case=Dat,Gen', 
+                    'e' => 'Case=Acc,Dat', 
+                    'd' => 'Case=Dat', 
+                    'b' => 'Case=Abl', 
+                    'i' => 'Case=Ins', 
+                    'l' => 'Case=Loc', 
+                    'v' => 'Case=Voc' },
+        :degree => {'p' => 'Degree=Pos', 
+                    'c' => 'Degree=Cmp', 
+                    's' => 'Degree=Sup' },
+        # The whole strength category is not universal
+        :strength => {'w' => 'Strength=Weak',
+                      's' => 'Strength=Strong'},
+        :inflection => {},
+      }
+    end
+  end
+end
+    

data/lib/proiel/cli/converters/conll-u/syntax.rb

@@ -0,0 +1,81 @@
+module PROIEL
+  module Converter
+    class CoNLLU
+      RELATION_MAPPING = {
+        "adnom" => "dep",
+        "adv" =>  [["advcl", lambda(&:clausal?) ],
+                   ["advmod", lambda { |x| x.adverb? or x.preposition? } ],
+                   ["advmod", lambda(&:adjectival?) ], # adjective for adverb
+                   ["nmod", lambda(&:nominal?) ],
+                   ["advmod", lambda { |x| true } ],
+                  ],
+        "ag" => "nmod", # add :agent" once defined
+        "apos" => [["name", lambda { |x| x.proper_noun? and x.head and x.head.proper_noun? } ],
+                   ["appos", lambda { |x| (x.nominal? or x.adjectival?) and x.head and x.head.nominal? } ],
+                   ["acl", lambda { |x| x.clausal? and x.head and x.head.nominal? } ],  # add :relcl ?
+                   # what to do about sentential appositions?
+                   ["advcl", lambda(&:clausal?) ],
+                   ["appos", lambda { |x| true } ],
+                  ],
+        "arg" => "dep",
+        "atr" => [["nummod", lambda(&:cardinal?) ],
+                  ["nmod", lambda(&:nominal?) ],
+                  ["acl", lambda { |x| x.clausal? } ],  # add :relcl?
+                  ["advmod", lambda { |x| x.head and x.head.clausal? } ],
+                  ["det", lambda(&:determiner?) ],
+                  ["amod", lambda { |x| true } ], #default
+                 ],
+        "aux" => [["det", lambda(&:determiner?) ],
+                  ["auxpass", lambda { |x| x.clausal? and x.head.passive?  } ],  
+                  ["aux", lambda(&:clausal?) ],
+                  ["neg", lambda(&:negation?) ],
+                  ["discourse", lambda { |x| x.particle? or x.interjection? } ],
+                  ["advmod", lambda { |x| x.adjectival? or x.adverb? } ], # or subjunction (? why did I write this?)
+                  ["cc", lambda(&:conjunction?) ],
+                  ["foreign", lambda(&:foreign?) ],
+                  # We need some more distinctions to get Gothic and Armenian. Introduce language in the treebank? (Read from xml)
+                  ["mark", lambda { |x| ['Pk', 'R-'].include? x.part_of_speech  } ], #reflexive as valency reducer, 'R-' as infinitive marker in Gothic
+                  ['amod', lambda { |x| x.preposition? } ], # Armenian DOM
+                  ['mwe', lambda { |x| ['Px', 'Pr'].include? x.part_of_speech } ], # NB there are a lot of bogus annotations with 'Px'
+                  
+                  # MISANNOTATION  IF A NOUN or a 'Pi' or a 'Pp' or a 'Ps'
+                 ],
+        "comp" => [['csubjpass', lambda { |x| x.head and x.head.passive? } ],
+                   ['csubj', lambda { |x| x.head and x.head.copula? } ],
+                   ['ccomp', lambda { |x| true } ],
+                  ],
+        "expl" => "expl",
+        "narg" => [['acl', lambda(&:clausal?) ],
+                   ['nmod', lambda(&:nominal?) ],
+                   ['nmod', lambda(&:adjectival?) ], # nominaliezed in this function
+                   ['nmod', lambda { |x| true } ],
+                  ],
+        "nonsub" => "dep",
+        "obj" => "dobj",
+        "obl" => [["advmod", lambda { |x| x.adverb? or x.preposition? } ], # though normally a preposition will be subordinate to its noun
+                  ["iobj", lambda(&:nominal?) ],# if nominal (NB check for presence of article!)
+                  ["iobj", lambda(&:adjectival?) ], # OBL adjectives are nominalized 
+                  ["advcl", lambda(&:clausal?) ], # this seems to happen with ad libros legendos etc. but check closer!
+                  ["iobj", lambda { |x| true } ],
+                 ],
+        "parpred" => "parataxis",
+        "part" => "nmod",
+        "per" => "dep",
+        "pid" => ["ERROR", lambda { |x| raise "Remaining pid edge!" } ],
+        "pred" => [["root", lambda(&:root?) ],
+                   ["ERROR", lambda { |x| raise "#{x.to_n} (head_id #{x.head_id}) is not a root!" }],
+                  ],
+        "rel" => "acl", # add :relcl?
+        "sub" => [["nsubjpass", lambda { |x| x.head and x.head.passive? } ],
+                  ["nsubj", lambda { |x| true }],
+                 ],
+        "voc" => "vocative",
+        "xadv" => [["advcl", lambda(&:clausal?)], #add :contr ?
+                   ["advmod", lambda { |x| true } ], # add :contr ?
+                  ],
+        "xobj" => "xcomp", # copula cases have already been taken care of
+        "xsub" => "xsub",
+      }
+    end
+  end
+end

data/lib/proiel/cli/converters/conll-x.rb

@@ -0,0 +1,66 @@
+module PROIEL
+  module Converter
+    # This converts to the CoNLL-X format as described on http://ilk.uvt.nl/conll/#dataformat.
+    class CoNLLX
+      class << self
+        def process(tb, options)
+          tb.sources.each do |source|
+            source.divs.each do |div|
+              div.sentences.each do |sentence|
+                id_to_number = {}
+
+                # Do not care about prodrop tokens
+                tk = sentence.tokens.reject { |t| t.empty_token_sort == 'P' }
+                
+                # Renumber to make the sequence continguous after prodrop tokens where left out
+                tk.map(&:id).each_with_index.each do |id, i|
+                  id_to_number[id] = i + 1
+                end
+
+                id_to_token = tk.inject({}) { |h, t| h.merge({t.id => t}) }
+
+                tk.each do |token|
+                  unless token.is_empty?
+                    this_number = id_to_number[token.id]
+                    head_number, relation = find_lexical_head_and_relation(id_to_number, id_to_token, token)
+                    form = token.form.gsub(/[[:space:]]/, '.')
+                    lemma = token.lemma.gsub(/[[:space:]]/, '.')
+                    pos_major = token.part_of_speech_hash[:major]
+                    pos_full = token.part_of_speech
+                    morphology = format_morphology(token)
+
+                    puts [this_number, form, lemma, pos_major, pos_full,
+                          morphology, head_number, relation, "_", "_"].join("\t")
+                  end
+                end
+
+                puts
+              end
+            end
+          end
+        end
+
+        def format_morphology(token)
+          token.morphology_hash.map do |k, v|
+            # Remove inflection tag unless when set to inflecting
+            if k == :inflection and v =='i'
+              nil
+            else
+              "#{k.upcase[0..3]}#{v}"
+            end
+          end.compact.join('|')
+        end
+
+        def find_lexical_head_and_relation(id_to_number, id_to_token, t, rel = '')
+          if t.is_root?
+            [0, rel + t.relation] # FIXME: may be empty token anyway
+          elsif id_to_token[t.head_id].has_content?
+            [id_to_number[t.head_id], rel + t.relation]
+          else
+            find_lexical_head_and_relation(id_to_number, id_to_token, id_to_token[t.head_id], rel + "#{t.relation}(#{id_to_number[t.head_id]})")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/proiel/cli/converters/lexc.rb

@@ -0,0 +1,36 @@
+module PROIEL
+  module Converter
+    # This converts part of speech and morphology to a lexc file.
+    class Lexc
+      class << self
+        def process(tb, options)
+          lexicon = {}
+
+          tb.sources.each do |source|
+            source.divs.each do |div|
+              div.sentences.each do |sentence|
+                sentence.tokens.each do |token|
+                  unless token.is_empty?
+                    lexicon[token.form] ||= []
+                    if options['morphology']
+                      lexicon[token.form] << [token.lemma, [token.part_of_speech, token.morphology].join].join(',')
+                    else
+                      lexicon[token.form] << [token.lemma, token.part_of_speech].join(',')
+                    end
+                  end
+                end
+              end
+            end
+          end
+
+          puts "LEXICON Root"
+          lexicon.sort.each do |form, tags|
+            tags.sort.uniq.each do |tag|
+              puts "  %s:%s #;" % [tag, form]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/proiel/cli/converters/proielxml.rb

@@ -0,0 +1,152 @@
+module PROIEL
+  module Converter
+    class PROIELXML
+      class << self
+        def process(tb, options)
+          builder = Builder::XmlMarkup.new(target: STDOUT, indent: 2)
+          builder.instruct! :xml, version: '1.0', encoding: 'UTF-8'
+          builder.proiel('export-time' => DateTime.now.xmlschema, 'schema-version' => '2.0') do
+            builder.annotation do
+              builder.relations do
+                tb.annotation_schema.relation_tags.each do |tag, value|
+                  attrs = { tag: tag }
+                  attrs.merge!(grab_features(value, %i(summary primary secondary)))
+                  builder.value(attrs)
+                end
+              end
+
+              builder.tag! 'parts-of-speech' do
+                tb.annotation_schema.part_of_speech_tags.each do |tag, value|
+                  attrs = { tag: tag }
+                  attrs.merge!(grab_features(value, %i(summary)))
+                  builder.value(attrs)
+                end
+              end
+
+              builder.morphology do
+                tb.annotation_schema.morphology_tags.each do |cat_tag, cat_values|
+                  builder.field(tag: cat_tag) do
+                    cat_values.each do |tag, value|
+                      attrs = { tag: tag }
+                      attrs.merge!(grab_features(value, %i(summary)))
+                      builder.value(attrs)
+                    end
+                  end
+                end
+              end
+
+              builder.tag! 'information-statuses' do
+                tb.annotation_schema.information_status_tags.each do |tag, value|
+                  attrs = { tag: tag }
+                  attrs.merge!(grab_features(value, %i(summary)))
+                  builder.value(attrs)
+                end
+              end
+            end
+
+            tb.sources.each do |source|
+              builder.source(id: source.id, language: source.language) do
+                PROIEL::Treebank::METADATA_ELEMENTS.each do |field|
+                  builder.tag!(field.to_s.gsub('_', '-'), source.send(field)) if source.send(field)
+                end
+
+                source.divs.each do |div|
+                  if include_div?(div, options)
+                    builder.div(grab_features(div, %i(), %i(presentation_before presentation_after))) do
+                      builder.title div.title if div.title
+
+                      div.sentences.each do |sentence|
+                        if include_sentence?(sentence, options)
+                          mandatory_features = %i(id)
+
+                          optional_features = [] # we do it this way to preserve the order of status and presentation_* so that diffing files is easier
+                          optional_features += %i(status) unless options['remove-status']
+                          optional_features += %i(presentation_before presentation_after)
+
+                          builder.sentence(grab_features(sentence, mandatory_features, optional_features)) do
+                            sentence.tokens.each do |token|
+                              next if token.empty_token_sort == 'P' and options['remove-information-structure']
+                              next if token.empty_token_sort == 'C' and options['remove-syntax']
+                              next if token.empty_token_sort == 'V' and options['remove-syntax']
+
+                              mandatory_features = %i(id)
+
+                              optional_features = %i(citation_part)
+                              optional_features += %i(lemma part_of_speech morphology) unless options['remove-morphology']
+                              optional_features += %i(head_id relation) unless options['remove-syntax']
+                              optional_features += %i(antecedent_id information_status contrast_group) unless options['remove-information-structure']
+
+                              unless token.is_empty?
+                                mandatory_features << :form
+                                optional_features += %i(presentation_before presentation_after foreign_ids)
+                              else
+                                mandatory_features << :empty_token_sort
+                              end
+
+                              attrs = grab_features(token, mandatory_features, optional_features)
+
+                              unless token.slashes.empty? or options['remove-syntax'] # this extra test avoids <token></token> style XML
+                                builder.token(attrs) do
+                                  token.slashes.each do |relation, target_id|
+                                    builder.slash("target-id": target_id, relation: relation)
+                                  end
+                                end
+                              else
+                                unless options['remove-syntax'] and token.is_empty?
+                                  builder.token(attrs)
+                                end
+                              end
+                            end
+                          end
+                        end
+                      end
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        def include_div?(div, options)
+          if options['remove-empty-divs']
+            div.sentences.any? { |sentence| include_sentence?(sentence, options) }
+          else
+            true
+          end
+        end
+
+        def include_sentence?(sentence, options)
+          case sentence.status
+          when :reviewed
+            true
+          when :annotated
+            not options['remove-not-reviewed']
+          else
+            not options['remove-not-reviewed'] and not options['remove-not-annotated']
+          end
+        end
+
+        def grab_features(obj, mandatory_features, optional_features = [])
+          attrs = {}
+
+          mandatory_features.each do |f|
+            v = obj.send(f)
+
+            attrs[f.to_s.gsub('_', '-')] = v
+          end
+
+          optional_features.each do |f|
+            v = obj.send(f)
+
+            if v and v.to_s != ''
+              attrs[f.to_s.gsub('_', '-')] = v
+            end
+          end
+
+          attrs
+        end
+      end
+    end
+  end
+end