proiel-cli 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cc539e4839fccb93166f5fb00309efafc88bdb25
-  data.tar.gz: 758707d18035ce10ecd59772031b6c01d703723e
+  metadata.gz: bfb4db333ae791d4171490c7b3d316364b205729
+  data.tar.gz: fd001265bdcc8e75e49fc1e2a188f7c63471bb62
 SHA512:
-  metadata.gz: cedb768615cbc1e6c2ecb1b09dc30f88f97d72df5a7545fd0a1087cf925432887c4838316609a6b5706f79f63cf9c35898e2ca64ce1d850e0ba264e99f6d1b3f
-  data.tar.gz: f105fcc4028036319efb2b4c7b3d9c5427a00c273166719f8256adc029f8ca1545514985ed814deea5c430682342cc10043ed94dc4c37de180db8e41717f957c
+  metadata.gz: 29cb26e28f22486db097b982d7e2cd4783f2b38d8d6c13fb06309fb25377791cd8248c88627b7d5b72c3adfe0e958c9696b2508fcb975e9a8fdf516a9825963b
+  data.tar.gz: b565292467456b10415039f0464bdfafcae5fecf55747dd3b9ec2473313ba08789d49431ecb06d649596bfd98031640481efccd9b5be58b6b565d13dcc3a1161

data/README.md

@@ -1,10 +1,10 @@
 # PROIEL command-line interface
 
-This is a command-line interface for manpulating PROIEL treebanks.
+This is a command-line interface for manipulating PROIEL treebanks.
 
 ## Installation
 
-Install as
+This library requires Ruby >= 2.1. Install as
 
 ```shell
 gem install proiel-cli

data/lib/proiel/cli/converters/proielxml.rb

@@ -88,7 +88,7 @@ module PROIEL
                               unless token.slashes.empty? or options['remove-syntax'] # this extra test avoids <token></token> style XML
                                 builder.token(attrs) do
                                   token.slashes.each do |relation, target_id|
-                                    builder.slash("target-id": target_id, relation: relation)
+                                    builder.slash(:"target-id" => target_id, relation: relation)
                                   end
                                 end
                               else

data/lib/proiel/cli/version.rb

@@ -1,5 +1,5 @@
 module PROIEL
   module CLI
-    VERSION = '0.1.0'
+    VERSION = '0.1.1'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: proiel-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Marius L. Jøhndal
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-07 00:00:00.000000000 Z
+date: 2015-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: builder
@@ -197,11 +197,6 @@ files:
 - contrib/proiel-maltparser-parse
 - contrib/proiel-maltparser-train
 - contrib/proiel-tnt-train
-- examples/decision-tree.rb
-- examples/dep-pos-cooccurrences.rb
-- examples/lint-rules.rb
-- examples/relation-as-disambiguator.rb
-- examples/word-occurrences.rb
 - lib/proiel/cli.rb
 - lib/proiel/cli/commands.rb
 - lib/proiel/cli/commands/convert.rb

data/examples/decision-tree.rb

@@ -1,41 +0,0 @@
-#!/usr/bin/env ruby
-#
-# Train a decision tree on the (head_relation, head_pos, head_lemma,
-# child_relation, child_pos, child_lemma) and then predict the child_relation
-# of an unknown child.
-#
-require 'colorize'
-require 'decisiontree'
-require 'proiel'
-
-if ARGV.length < 1
-  STDERR.puts "Usage: #{$0} treebank-files(s)"
-  exit 1
-end
-
-tb = PROIEL::Treebank.new
-tb.load_from_xml(ARGV)
-
-tokens = {}
-
-tb.sources.each do |source|
-  source.tokens.each do |token|
-    tokens[token.id.to_i] = [token.relation, token.part_of_speech, token.lemma, token.head_id]
-  end
-end
-
-training_data = tokens.map do |_, (child_relation, child_pos, child_lemma, head_id)|
-    if head_id
-      head = tokens[head_id.to_i]
-      head_relation, head_pos, head_lemma, _ = *head
-
-      [head_pos || '', head_lemma || '', head_relation, child_pos || '', child_lemma || '', child_relation]
-    end
-  end.compact
-
-attributes = %w(head_pos head_lemma head_relation child_pos child_lemma)
-dr = DecisionTree::ID3Tree.new(attributes, training_data, 'pred', :discrete)
-dr.train
-dr.save_to_file("dr.marshal")
-
-p dr.predict(["Ne", "Gallia", "sub", "Px", "omnis"])

data/examples/dep-pos-cooccurrences.rb

@@ -1,84 +0,0 @@
-#!/usr/bin/env ruby
-require 'proiel'
-require 'colorize'
-require 'terminal-table'
-
-if ARGV.length < 1
-  STDERR.puts "Usage: #{$0} treebank-files(s)"
-  exit 1
-end
-
-tb = PROIEL::Treebank.new
-tb.load_from_xml(ARGV)
-
-# Present by POS
-relations = tb.annotation.relation_tags.keys
-
-c = {}
-tb.sources.each do |s|
-  s.tokens.each do |t|
-    next if t.pos.nil? or t.relation.nil?
-
-    c[t.pos] ||= {}
-    c[t.pos][t.relation] ||= 0
-    c[t.pos][t.relation] += 1
-  end
-end
-
-rows = []
-c.sort_by(&:first).each do |pos, d|
-  total = d.inject(0) { |a, (k, v)| a + v }
-
-  rows << [pos] + relations.map do |r|
-    n = d[r ? r.to_s : nil]
-
-    if n and n < total * 0.001
-      n.to_s.red
-    elsif n and n > total * 0.999
-      n.to_s.green
-    else
-      n
-    end
-  end
-end
-
-table = Terminal::Table.new headings: ['Part of speech'] + relations, rows: rows
-puts table
-puts "(red = relation occurs for less than 0.1% of tokens with this POS; green = relation occurs for more than 99.9% of tokens with this POS)"
-puts
-
-# Present by relation
-poses = tb.annotation.part_of_speech_tags.keys
-
-c = {}
-
-tb.sources.each do |s|
-  s.tokens.each do |t|
-    next if t.pos.nil? or t.relation.nil?
-
-    c[t.relation] ||= {}
-    c[t.relation][t.pos] ||= 0
-    c[t.relation][t.pos] += 1
-  end
-end
-
-rows = []
-c.sort_by(&:first).each do |relation, d|
-  total = d.inject(0) { |a, (k, v)| a + v }
-
-  rows << [relation] + poses.map do |r|
-    n = d[r ? r.to_s : nil]
-
-    if n and n < total * 0.001
-      n.to_s.red
-    elsif n and n > total * 0.999
-      n.to_s.green
-    else
-      n
-    end
-  end
-end
-
-table = Terminal::Table.new headings: ['Relation'] + poses, rows: rows
-puts table
-puts "(red = POS occurs for less than 0.1% of tokens with this relation; green = POS occurs for more than 99.9% of tokens with this relation)"

data/examples/lint-rules.rb

@@ -1,174 +0,0 @@
-#!/usr/bin/env ruby
-#
-# Very simple testing of implicational feature rules. Example rules only
-# apply to Latin.
-#
-require 'colorize'
-require 'proiel'
-
-VIOLATIONS = {}
-
-def report_violation(token, message)
-  VIOLATIONS[message] ||= []
-  VIOLATIONS[message] << token
-end
-
-def test_token(token, rules, dependent_rules)
-  rules.each do |match_features, test_alternatives|
-    f = token.features + ["\"#{token.form}\""]
-
-    if (match_features - f).empty?
-      unless test_alternatives.any? { |test_alternative| token.features.include?(test_alternative) }
-        report_violation(token, "#{match_features.join(' ')}")
-      end
-    end
-  end
-
-  dependent_rules.each do |match_features, test_alternatives|
-    f = token.features + ["\"#{token.form}\""]
-
-    if (match_features - f).empty?
-      t = token.children.all? do |dependent|
-          test_alternatives.any? { |test_alternative| dependent.features.include?(test_alternative) }
-        end
-
-      unless t
-        report_violation(token, "#{match_features.join(' ')} → dependents()")
-      end
-    end
-  end
-end
-
-def load_rules
-  rules = {}
-  dependent_rules = {}
-
-  DATA.each do |rule|
-    rule.chomp!
-    rule.sub!(/\s*#.*$/, '')
-
-    next if rule.empty?
-
-    match_features, test = rule.split(/\s*→\s*/)
-    match_features = match_features.split(/\s+/)
-
-    if test[/\s*dependents\(([^)]*)\)\s*/]
-      dependent_rules[match_features] ||= []
-      dependent_rules[match_features] << $1
-      test.sub!(/\s*dependents\([^)]*\)\s*/, '')
-    end
-
-    if test != ''
-      rules[match_features] ||= []
-      rules[match_features] << test
-    end
-  end
-
-  [rules, dependent_rules]
-end
-
-if ARGV.length < 1
-  STDERR.puts "Usage: #{$0} treebank-files(s)"
-  exit 1
-end
-
-tb = PROIEL::Treebank.new
-tb.load_from_xml(ARGV)
-
-rules, dependent_rules = load_rules
-
-tb.sources.each do |source|
-  source.sentences.each do |sentence|
-    if sentence.status == 'reviewed'
-      sentence.tokens.each do |token|
-        test_token(token, rules, dependent_rules)
-      end
-    end
-  end
-end
-
-base_url = 'http://foni.uio.no:3000'
-
-puts "<h1>PROIEL lint report</h1>"
-
-VIOLATIONS.each do |rule, tokens|
-  puts "<h2>#{rule}</h2><ul>"
-  tokens.each do |token|
-    puts "<li>Token <a href='#{base_url}/tokens/#{token.id}'>#{token.id}</a> in sentence <a href='#{base_url}/sentences/#{token.sentence.id}'>#{token.sentence.id}</a></li>"
-  end
-  puts "</ul>"
-end
-
-__END__
-
-# Gerundives
-gdv nom → xobj    # modal gerundive heading a main clause
-
-gdv acc → comp    # modal gerundive heading an AcI, or in the _curo faciendum_ type
-gdv acc → xobj    # modal gerundive heading an AcI with an overt auxiliary
-gdv acc → obl     # as argument of a preposition
-gdv acc → xadv    # in the _do librum legendum_ type
-
-gdv gen → atr     # in the _tempus dicendi_ type
-gdv gen → narg    # in the _facultas dicendi_ type
-
-gdv abl → obl     # as argument of a preposition
-gdv abl → abl     # in circumstantial adjuncts of various types
-
-# Gerunds
-ger nom → 0       # invalid case for a gerundive
-
-ger acc → obl     # as argument of a preposition
-
-ger gen → atr     # in the _tempus dicendi_ type
-ger gen → narg    # in the _facultas dicendi_ type
-
-ger abl → obl     # as argument of a preposition
-ger abl → abl     # in circumstantial adjuncts of various types
-
-# Reflexive pronouns
-persrefl nom → 0
-
-persrefl acc → sub
-persrefl acc → obj
-persrefl acc → obl
-
-persrefl dat → obl
-persrefl dat → adv
-persrefl dat → ag
-
-persrefl abl → obl
-persrefl abl → sub
-
-persrefl "se" → acc
-persrefl "se" → abl
-
-persrefl "sese" → acc
-persrefl "sese" → abl
-
-persrefl "sibi" → dat
-
-# Personal pronouns
-perspron nom → sub
-
-perspron acc → sub
-perspron acc → obj
-perspron acc → obl
-
-perspron dat → obl
-perspron dat → adv
-perspron dat → ag
-
-perspron abl → obl
-perspron abl → sub
-
-# The dependent of the complementisers _ut_ and _ne_ should be a PRED or an AUX
-subj "ut" → dependents(pred)   # the standard case, a predicate heading a clause
-subj "ut" → dependents(aux)    # some particle-like material dependent on the complementiser
-
-subj "ne" → dependents(pred)
-subj "ne" → dependents(aux)
-
-# Particles and adverbs
-"iam" → adverb adv
-"iam" → adverb aux # possibly

data/examples/relation-as-disambiguator.rb

@@ -1,134 +0,0 @@
-#!/usr/bin/env ruby
-#
-# Does the dependency relation suffice to disambiguate ambiguous morphology?
-#
-require 'colorize'
-require 'proiel'
-
-if ARGV.length < 1
-  STDERR.puts "Usage: #{$0} treebank-files(s)"
-
-  exit 1
-end
-
-tb = PROIEL::Treebank.new
-tb.load_from_xml(ARGV)
-
-# Harvest morphology
-form_hash = {}
-
-tb.sources.reject { |source| source.language != language_tag }.each do |source|
-  source.tokens.each do |token|
-    next unless token.form and token.pos and token.morphology and token.relation
-
-    # TODO: problem with using token.form is that sentence-initial words are sometimes capitalised
-    relation_hash = (form_hash[token.form] ||= {})
-    pos_hash = (relation_hash[token.relation] ||= {})
-    morphology_hash = (pos_hash[token.pos] ||= {})
-    morphology_hash[token.morphology] ||= 0
-    morphology_hash[token.morphology] += 1
-  end
-end
-
-# Calculate by unique forms first
-unique_pos = 0
-relation_predicts_pos = 0
-relation_does_not_predict_pos = 0
-
-unique_morphology = 0
-relation_predicts_morphology = 0
-relation_does_not_predict_morphology = 0
-
-form_hash.each do |form, h|
-  number_of_poses = 0
-  number_of_morphologies = 0
-
-  h.each do |_, i|
-    i.each do |_, j|
-      number_of_poses += 1
-      j.each do |_, k|
-        number_of_morphologies += 1
-      end
-    end
-  end
-
-  if number_of_poses == 1
-    unique_pos += 1
-  elsif h.all? { |_, i| i.keys.count == 1 }
-    relation_predicts_pos += 1
-  else
-    relation_does_not_predict_pos += 1
-  end
-
-  if number_of_morphologies == 1
-    unique_morphology += 1
-  elsif h.all? { |_, i| i.all? { |_, j| j.keys.count == 1 } }
-    relation_predicts_morphology += 1
-  else
-    relation_does_not_predict_morphology += 1
-  end
-end
-
-puts "By unique forms (types)"
-puts "======================="
-puts "Forms with a unique POS:                               #{unique_pos}"
-puts "Forms whose relation predicts its POS:                 #{relation_predicts_pos}"
-puts "Forms whose relation does not predict its POS:         #{relation_does_not_predict_pos}"
-
-puts "Forms with a unique morphology:                        #{unique_morphology}"
-puts "Forms whose relation predicts its morphology:          #{relation_predicts_morphology}"
-puts "Forms whose relation does not predict its morphology:  #{relation_does_not_predict_morphology}"
-
-# Calculate by actual number of occurrences
-unique_pos = 0
-relation_predicts_pos = 0
-relation_does_not_predict_pos = 0
-
-unique_morphology = 0
-relation_predicts_morphology = 0
-relation_does_not_predict_morphology = 0
-
-form_hash.each do |form, h|
-  n = 0
-  number_of_poses = 0
-  number_of_morphologies = 0
-
-  h.each do |_, i|
-    i.each do |_, j|
-      number_of_poses += 1
-      j.each do |_, k|
-        number_of_morphologies += 1
-        n += k
-      end
-    end
-  end
-
-  if number_of_poses == 1
-    unique_pos += n
-  elsif h.all? { |_, i| i.keys.count == 1 }
-    relation_predicts_pos += n
-  else
-    relation_does_not_predict_pos += n
-  end
-
-  if number_of_morphologies == 1
-    unique_morphology += n
-  elsif h.all? { |_, i| i.all? { |_, j| j.keys.count == 1 } }
-    relation_predicts_morphology += n
-  else
-    relation_does_not_predict_morphology += n
-  end
-end
-
-puts
-puts "By occurrences of forms (tokens)"
-puts "================================"
-puts "Forms with a unique POS:                               #{unique_pos}"
-puts "Forms whose relation predicts its POS:                 #{relation_predicts_pos}"
-puts "Forms whose relation does not predict its POS:         #{relation_does_not_predict_pos}"
-
-puts "Forms with a unique morphology:                        #{unique_morphology}"
-puts "Forms whose relation predicts its morphology:          #{relation_predicts_morphology}"
-puts "Forms whose relation does not predict its morphology:  #{relation_does_not_predict_morphology}"
-
-exit 0

data/examples/word-occurrences.rb

@@ -1,30 +0,0 @@
-#!/usr/bin/env ruby
-#
-# Word form occurrence extraction
-#
-require 'colorize'
-require 'proiel'
-
-if ARGV.length < 1
-  STDERR.puts "Usage: #{$0} treebank-files(s)"
-
-  exit 1
-end
-
-tb = PROIEL::Treebank.new
-tb.load_from_xml(ARGV)
-
-form_index = {}
-
-tb.sources.each do |source|
-  source.tokens.each do |token|
-    unless token.form.nil?
-      form_index[token.form] ||= []
-      form_index[token.form] << [source.id, token.id].join(':')
-    end
-  end
-end
-
-form_index.sort_by(&:first).each do |form, ids|
-  puts "#{form}: #{ids.join(', ')}"
-end