proiel-cli 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cc539e4839fccb93166f5fb00309efafc88bdb25
+  data.tar.gz: 758707d18035ce10ecd59772031b6c01d703723e
+SHA512:
+  metadata.gz: cedb768615cbc1e6c2ecb1b09dc30f88f97d72df5a7545fd0a1087cf925432887c4838316609a6b5706f79f63cf9c35898e2ca64ce1d850e0ba264e99f6d1b3f
+  data.tar.gz: f105fcc4028036319efb2b4c7b3d9c5427a00c273166719f8256adc029f8ca1545514985ed814deea5c430682342cc10043ed94dc4c37de180db8e41717f957c

data/LICENSE

@@ -0,0 +1,23 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Marius L. Jøhndal
+Copyright (c) 2015 Dag Haug
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,34 @@
+# PROIEL command-line interface
+
+This is a command-line interface for manpulating PROIEL treebanks.
+
+## Installation
+
+Install as
+
+```shell
+gem install proiel-cli
+```
+
+## Using the command-line interface
+
+The gem includes a command-line utility `proiel` for a number of routine tasks.
+`proiel info`, for example, displays metadata and some brief statistics, and
+`proiel convert conll` converts the treebank to CoNLL format. Use `proiel
+--help` for further examples and usage instructions.
+
+## Development
+
+Check out the git repository from github and run `bin/setup` to install
+development dependencies. Then run `rake` to run the tests.
+
+To install a development version of this gem, run `bundle exec rake install`.
+To release a new version, update the version number in `version.rb`, and then
+run `bundle exec rake release`, which will create a git tag for the version,
+push git commits and tags, and push the gem to
+[rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/proiel/proiel-cli.

data/bin/proiel

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+STDOUT.sync = true
+
+$:.unshift File.join(File.dirname(__FILE__), *%w{ .. lib })
+
+require 'proiel/cli'
+
+Mercenary.program(:proiel) do |p|
+  p.version PROIEL::VERSION
+  p.description 'proiel is a command-line interface for PROIEL treebanks'
+  p.syntax 'proiel <subcommand> [options]'
+
+  p.option 'verbose', '-V', '--verbose', 'Display verbose information'
+
+  PROIEL::Command.subclasses.each { |c| c.init_with_program(p) }
+
+  p.action do |args, options|
+    if args.empty?
+      puts p
+    else
+      unless p.has_command?(args.first)
+        STDERR.puts 'Invalid command. Use --help for more information.'
+        exit 1
+      end
+    end
+  end
+end

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/contrib/proiel-giza-train

@@ -0,0 +1,6 @@
+#!/bin/bash
+ln -sf "$1" a
+ln -sf "$2" b
+plain2snt a b
+snt2cooc a.vcb b.vcb a_b.snt > corp.cooc
+GIZA++ -S a.vcb -T b.vcb -C a_b.snt -CoocurrenceFile corp.cooc -o alignment > alignment.log

data/contrib/proiel-lexc-compile

@@ -0,0 +1,18 @@
+#!/bin/bash
+tmpdir=$(mktemp -dt "$0.XXXXXXXXXX")
+lexcfile=${tmpdir}/output.lexc
+scriptfile=${tmpdir}/output.script
+fstfile=output.fst
+
+proiel=../bin/proiel
+foma=foma
+#foma=xfst
+
+../bin/proiel convert lexc -V $@ > ${lexcfile} || exit 1
+echo "read lexc ${lexcfile}" > ${scriptfile}
+echo "save stack ${fstfile}" >> ${scriptfile}
+
+${foma} -f ${scriptfile}
+
+echo
+echo "Generated transducer is ${fstfile}. Try 'echo wordform | flookup ${fstfile}' to test."

data/contrib/proiel-maltparser-parse

@@ -0,0 +1,2 @@
+#!/bin/sh
+java -jar maltparser-1.8.jar -c test -i untagged.conll -o tagged.conll -m parse

data/contrib/proiel-maltparser-train

@@ -0,0 +1,6 @@
+#!/bin/sh
+java \
+  -jar maltparser-1.8.jar \
+  -c test \
+  -m learn \
+  -i $1

data/contrib/proiel-tnt-train

@@ -0,0 +1,15 @@
+#!/bin/bash
+tmpdir=$(mktemp -dt "$0.XXXXXXXXXX")
+tntfile=${tmpdir}/output.t
+#fstfile=output.fst
+
+proiel=../bin/proiel
+tnt=tnt
+#tnt=hunpos
+
+../bin/proiel convert tnt -V $@ > ${tntfile} || exit 1
+
+tnt-para ${tntfile} && mv -vi ${tmpdir}/output.{lex,123} .
+
+echo
+echo "Generated parameter files are output.lex and output.123. Try 'tnt output mycorpus.t' to test."

data/examples/decision-tree.rb

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+#
+# Train a decision tree on the (head_relation, head_pos, head_lemma,
+# child_relation, child_pos, child_lemma) and then predict the child_relation
+# of an unknown child.
+#
+require 'colorize'
+require 'decisiontree'
+require 'proiel'
+
+if ARGV.length < 1
+  STDERR.puts "Usage: #{$0} treebank-files(s)"
+  exit 1
+end
+
+tb = PROIEL::Treebank.new
+tb.load_from_xml(ARGV)
+
+tokens = {}
+
+tb.sources.each do |source|
+  source.tokens.each do |token|
+    tokens[token.id.to_i] = [token.relation, token.part_of_speech, token.lemma, token.head_id]
+  end
+end
+
+training_data = tokens.map do |_, (child_relation, child_pos, child_lemma, head_id)|
+    if head_id
+      head = tokens[head_id.to_i]
+      head_relation, head_pos, head_lemma, _ = *head
+
+      [head_pos || '', head_lemma || '', head_relation, child_pos || '', child_lemma || '', child_relation]
+    end
+  end.compact
+
+attributes = %w(head_pos head_lemma head_relation child_pos child_lemma)
+dr = DecisionTree::ID3Tree.new(attributes, training_data, 'pred', :discrete)
+dr.train
+dr.save_to_file("dr.marshal")
+
+p dr.predict(["Ne", "Gallia", "sub", "Px", "omnis"])

data/examples/dep-pos-cooccurrences.rb

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+require 'proiel'
+require 'colorize'
+require 'terminal-table'
+
+if ARGV.length < 1
+  STDERR.puts "Usage: #{$0} treebank-files(s)"
+  exit 1
+end
+
+tb = PROIEL::Treebank.new
+tb.load_from_xml(ARGV)
+
+# Present by POS
+relations = tb.annotation.relation_tags.keys
+
+c = {}
+tb.sources.each do |s|
+  s.tokens.each do |t|
+    next if t.pos.nil? or t.relation.nil?
+
+    c[t.pos] ||= {}
+    c[t.pos][t.relation] ||= 0
+    c[t.pos][t.relation] += 1
+  end
+end
+
+rows = []
+c.sort_by(&:first).each do |pos, d|
+  total = d.inject(0) { |a, (k, v)| a + v }
+
+  rows << [pos] + relations.map do |r|
+    n = d[r ? r.to_s : nil]
+
+    if n and n < total * 0.001
+      n.to_s.red
+    elsif n and n > total * 0.999
+      n.to_s.green
+    else
+      n
+    end
+  end
+end
+
+table = Terminal::Table.new headings: ['Part of speech'] + relations, rows: rows
+puts table
+puts "(red = relation occurs for less than 0.1% of tokens with this POS; green = relation occurs for more than 99.9% of tokens with this POS)"
+puts
+
+# Present by relation
+poses = tb.annotation.part_of_speech_tags.keys
+
+c = {}
+
+tb.sources.each do |s|
+  s.tokens.each do |t|
+    next if t.pos.nil? or t.relation.nil?
+
+    c[t.relation] ||= {}
+    c[t.relation][t.pos] ||= 0
+    c[t.relation][t.pos] += 1
+  end
+end
+
+rows = []
+c.sort_by(&:first).each do |relation, d|
+  total = d.inject(0) { |a, (k, v)| a + v }
+
+  rows << [relation] + poses.map do |r|
+    n = d[r ? r.to_s : nil]
+
+    if n and n < total * 0.001
+      n.to_s.red
+    elsif n and n > total * 0.999
+      n.to_s.green
+    else
+      n
+    end
+  end
+end
+
+table = Terminal::Table.new headings: ['Relation'] + poses, rows: rows
+puts table
+puts "(red = POS occurs for less than 0.1% of tokens with this relation; green = POS occurs for more than 99.9% of tokens with this relation)"

data/examples/lint-rules.rb

@@ -0,0 +1,174 @@
+#!/usr/bin/env ruby
+#
+# Very simple testing of implicational feature rules. Example rules only
+# apply to Latin.
+#
+require 'colorize'
+require 'proiel'
+
+VIOLATIONS = {}
+
+def report_violation(token, message)
+  VIOLATIONS[message] ||= []
+  VIOLATIONS[message] << token
+end
+
+def test_token(token, rules, dependent_rules)
+  rules.each do |match_features, test_alternatives|
+    f = token.features + ["\"#{token.form}\""]
+
+    if (match_features - f).empty?
+      unless test_alternatives.any? { |test_alternative| token.features.include?(test_alternative) }
+        report_violation(token, "#{match_features.join(' ')}")
+      end
+    end
+  end
+
+  dependent_rules.each do |match_features, test_alternatives|
+    f = token.features + ["\"#{token.form}\""]
+
+    if (match_features - f).empty?
+      t = token.children.all? do |dependent|
+          test_alternatives.any? { |test_alternative| dependent.features.include?(test_alternative) }
+        end
+
+      unless t
+        report_violation(token, "#{match_features.join(' ')} → dependents()")
+      end
+    end
+  end
+end
+
+def load_rules
+  rules = {}
+  dependent_rules = {}
+
+  DATA.each do |rule|
+    rule.chomp!
+    rule.sub!(/\s*#.*$/, '')
+
+    next if rule.empty?
+
+    match_features, test = rule.split(/\s*→\s*/)
+    match_features = match_features.split(/\s+/)
+
+    if test[/\s*dependents\(([^)]*)\)\s*/]
+      dependent_rules[match_features] ||= []
+      dependent_rules[match_features] << $1
+      test.sub!(/\s*dependents\([^)]*\)\s*/, '')
+    end
+
+    if test != ''
+      rules[match_features] ||= []
+      rules[match_features] << test
+    end
+  end
+
+  [rules, dependent_rules]
+end
+
+if ARGV.length < 1
+  STDERR.puts "Usage: #{$0} treebank-files(s)"
+  exit 1
+end
+
+tb = PROIEL::Treebank.new
+tb.load_from_xml(ARGV)
+
+rules, dependent_rules = load_rules
+
+tb.sources.each do |source|
+  source.sentences.each do |sentence|
+    if sentence.status == 'reviewed'
+      sentence.tokens.each do |token|
+        test_token(token, rules, dependent_rules)
+      end
+    end
+  end
+end
+
+base_url = 'http://foni.uio.no:3000'
+
+puts "<h1>PROIEL lint report</h1>"
+
+VIOLATIONS.each do |rule, tokens|
+  puts "<h2>#{rule}</h2><ul>"
+  tokens.each do |token|
+    puts "<li>Token <a href='#{base_url}/tokens/#{token.id}'>#{token.id}</a> in sentence <a href='#{base_url}/sentences/#{token.sentence.id}'>#{token.sentence.id}</a></li>"
+  end
+  puts "</ul>"
+end
+
+__END__
+
+# Gerundives
+gdv nom → xobj    # modal gerundive heading a main clause
+
+gdv acc → comp    # modal gerundive heading an AcI, or in the _curo faciendum_ type
+gdv acc → xobj    # modal gerundive heading an AcI with an overt auxiliary
+gdv acc → obl     # as argument of a preposition
+gdv acc → xadv    # in the _do librum legendum_ type
+
+gdv gen → atr     # in the _tempus dicendi_ type
+gdv gen → narg    # in the _facultas dicendi_ type
+
+gdv abl → obl     # as argument of a preposition
+gdv abl → abl     # in circumstantial adjuncts of various types
+
+# Gerunds
+ger nom → 0       # invalid case for a gerundive
+
+ger acc → obl     # as argument of a preposition
+
+ger gen → atr     # in the _tempus dicendi_ type
+ger gen → narg    # in the _facultas dicendi_ type
+
+ger abl → obl     # as argument of a preposition
+ger abl → abl     # in circumstantial adjuncts of various types
+
+# Reflexive pronouns
+persrefl nom → 0
+
+persrefl acc → sub
+persrefl acc → obj
+persrefl acc → obl
+
+persrefl dat → obl
+persrefl dat → adv
+persrefl dat → ag
+
+persrefl abl → obl
+persrefl abl → sub
+
+persrefl "se" → acc
+persrefl "se" → abl
+
+persrefl "sese" → acc
+persrefl "sese" → abl
+
+persrefl "sibi" → dat
+
+# Personal pronouns
+perspron nom → sub
+
+perspron acc → sub
+perspron acc → obj
+perspron acc → obl
+
+perspron dat → obl
+perspron dat → adv
+perspron dat → ag
+
+perspron abl → obl
+perspron abl → sub
+
+# The dependent of the complementisers _ut_ and _ne_ should be a PRED or an AUX
+subj "ut" → dependents(pred)   # the standard case, a predicate heading a clause
+subj "ut" → dependents(aux)    # some particle-like material dependent on the complementiser
+
+subj "ne" → dependents(pred)
+subj "ne" → dependents(aux)
+
+# Particles and adverbs
+"iam" → adverb adv
+"iam" → adverb aux # possibly