myaso 0.4.0

data/spec/lexicon_spec.rb

@@ -0,0 +1,84 @@
+# encoding: utf-8
+
+require_relative 'spec_helper'
+
+describe Myaso::Lexicon do
+  subject { Myaso::Lexicon.new }
+
+  describe '#new' do
+    it 'should be empty' do
+      subject.table.must_be_empty
+    end
+
+    it 'should not initialize @tags' do
+      subject.instance_variable_defined?(:@tags).must_equal false
+    end
+  end
+
+  describe '#[]' do
+    it 'should treat unknown words as zeroes' do
+      subject['lopata'].must_equal 0
+    end
+
+    it 'should treat unknown words and tags as zeroes' do
+      subject['lopata', 'dno'].must_equal 0
+    end
+
+    it 'should modify a word' do
+      subject['lopata'] = 1
+      subject['lopata'].must_equal 1
+    end
+
+    it 'should modify a word with tag' do
+      subject['lopata', 'dno'] = 2
+      subject['lopata', 'dno'].must_equal 2
+    end
+  end
+
+  describe '#tags' do
+    it 'should perform lazy initialization' do
+      subject.instance_variable_defined?(:@tags).must_equal false
+      subject.tags
+      subject.instance_variable_get(:@tags).wont_be_nil
+    end
+
+    it 'should collect global tag counts' do
+      subject['lopata', 'dno'] = 1
+      subject['lopata', 'bydlow'] = 2
+      subject.tags.must_equal 'dno' => 1, 'bydlow' => 2
+    end
+
+    it 'should be invalidated after the value assignment' do
+      subject.tags.must_be_empty
+      subject['lopata', 'dno'] = 1
+      subject.tags.must_equal 'dno' => 1
+    end
+
+    it 'should return tags of the given word' do
+      subject.tags('lopata').must_be_empty
+      subject['lopata', 'dno'] = 1
+      subject.tags('lopata').must_equal %w(dno)
+    end
+  end
+
+  describe '#each' do
+    it 'should iterate over the internal table' do
+      subject.each.to_a.must_equal subject.table.to_a
+    end
+  end
+
+  describe '#==' do
+    let(:other) { Myaso::Lexicon.new }
+
+    it 'should be equal to a new instance when not modified' do
+      subject.must_equal other
+    end
+
+    it 'should check equality by internal tables' do
+      subject['lopata', 'dno'] = 1
+      subject.wont_equal other
+      other['lopata', 'dno'] = 1
+      subject.must_equal other
+    end
+  end
+end

data/spec/mystem_spec.rb

@@ -0,0 +1,81 @@
+# encoding: utf-8
+
+require_relative 'spec_helper'
+
+describe Myaso::Mystem do
+  describe 'analysis of dictionary words' do
+    subject { Myaso::Mystem.analyze('СТАЛИ') }
+
+    it 'is ambiguous' do
+      subject.length.must_equal 2
+    end
+
+    it 'is a dictionary word' do
+      subject.each { |s| s.quality.must_equal :dictionary }
+    end
+
+    it 'lemmatizes' do
+      subject.map(&:lemma).sort!.must_equal %w(сталь становиться)
+    end
+
+    it 'normalizes' do
+      subject.each { |s| s.form.must_equal 'стали' }
+    end
+
+    it 'analyzes' do
+      subject.map { |s| s.msd.pos.to_s }.sort!.must_equal %w(noun verb)
+    end
+  end
+
+  describe 'analysis of bastard words' do
+    subject { Myaso::Mystem.analyze('дОлБоЯщЕрА') }
+
+    it 'is unambiguous' do
+      subject.length.must_equal 1
+    end
+
+    it 'is really a dictionary word' do
+      subject.first.quality.must_equal :bastard
+    end
+
+    it 'lemmatizes' do
+      subject.first.lemma.must_equal 'долбоящер'
+    end
+
+    it 'normalizes' do
+      subject.first.form.must_equal 'долбоящера'
+    end
+
+    it 'analyzes' do
+      subject.first.msd.pos.must_equal :noun
+    end
+  end
+
+  describe 'form enumeration' do
+    let(:lemma) { Myaso::Mystem.analyze('человеком').first }
+
+    subject { Myaso::Mystem.forms('человеком', 3890) }
+
+    it 'enumerates forms' do
+      subject.length.must_equal 14
+    end
+
+    it 'works for lemmas' do
+      subject.must_equal lemma.forms
+    end
+  end
+
+  describe 'inflection' do
+    let(:lemma) { Myaso::Mystem.analyze('людьми').first }
+
+    subject { lemma.inflect(:number => :plural, :case => :dative) }
+
+    it 'is ambiguous' do
+      subject.length.must_equal 2
+    end
+
+    it 'inflects' do
+      subject.map!(&:form).sort!.must_equal %w(людям человекам)
+    end
+  end
+end

data/spec/ngrams_spec.rb

@@ -0,0 +1,97 @@
+# encoding: utf-8
+
+require_relative 'spec_helper'
+
+describe Myaso::Ngrams do
+  let(:tags) { %w(D V N) }
+  let(:unigrams) { tags }
+  let(:bigrams) { tags + [nil] }
+  let(:trigrams) { tags + [nil] }
+
+  subject { Myaso::Ngrams.new }
+
+  describe '#new' do
+    it 'should be full of zeroes' do
+      unigrams.each do |u|
+        bigrams.each do |b|
+          trigrams.each do |t|
+            subject[u, b, t].must_equal 0
+          end
+        end
+      end
+    end
+
+    it 'should be empty' do
+      subject.unigrams_count.must_equal 0
+    end
+  end
+
+  describe '#[]' do
+    it 'should treat unset unigrams as zeroes' do
+      subject['D'].must_equal 0
+    end
+
+    it 'should treat unset bigrams as zeroes' do
+      subject['D', 'V'].must_equal 0
+    end
+
+    it 'should treat unset trigrams as zeroes' do
+      subject['D', 'V', 'N'].must_equal 0
+    end
+
+    it 'should modify an unigram' do
+      subject['D'] = 1
+      subject['D'].must_equal 1
+      subject.unigrams_count.must_equal 1
+    end
+
+    it 'should modify a bigram' do
+      subject['D', 'N'] = 2
+      subject['D', 'N'].must_equal 2
+    end
+
+    it 'should modify a trigram' do
+      subject['D', 'N', 'V'] = 3
+      subject['D', 'N', 'V'].must_equal 3
+    end
+  end
+
+  describe '#==' do
+    let(:other) { Myaso::Ngrams.new }
+
+    it 'should be equal to a new instance when not modified' do
+      subject.must_equal other
+    end
+
+    it 'should check equality by internal tables' do
+      subject['D', 'N', 'V'] = 1
+      subject.wont_equal other
+      other['D', 'N', 'V'] = 1
+      subject.must_equal other
+    end
+  end
+
+  describe '#each' do
+    before do
+      subject['D'] = 1
+      subject['N'] = 2
+      subject['D', 'N'] = 3
+      subject['V', 'D'] = 4
+      subject['D', 'N', 'V'] = 5
+      subject['N', 'V', 'D'] = 6
+    end
+
+    it 'should iterate over the internal table' do
+      subject.each.to_a.must_equal([["D", {nil=>{nil=>1},
+        "N"=>{nil=>3, "V"=>5}}], ["N", {nil=>{nil=>2}, "V"=>{"D"=>6}}],
+        ["V", {"D"=>{nil=>4}}]])
+    end
+
+    it 'should enumerate over trigrams' do
+      Array.new.tap do |trigrams|
+        subject.each_trigram { |trigram| trigrams << trigram }
+        trigrams.must_equal [[%w(D N V), 5], [%w(N V D), 6]]
+      end
+    end
+  end
+end

data/spec/pi_table_spec.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+
+require_relative 'spec_helper'
+
+describe Myaso::PiTable do
+  let(:length) { 10 }
+  let(:tags) { %w(D V N) }
+
+  subject { Myaso::PiTable.new }
+
+  describe '#new' do
+    it 'should be full of nils' do
+      length.times do |i|
+        tags.each do |u|
+          tags.each do |v|
+            subject[i, u, v].must_be_nil
+          end
+        end
+      end
+    end
+
+    describe 'with default value' do
+      let(:default) { 0 }
+
+      subject { Myaso::PiTable.new(default) }
+
+      it 'should be full of zeros' do
+        length.times do |i|
+          tags.each do |u|
+            tags.each do |v|
+              subject[i, u, v].must_equal 0
+            end
+          end
+        end
+      end
+    end
+  end
+
+  describe '#[]' do
+    it 'should get and set a tuple' do
+      subject[0, 'D', 'N'] = 1
+      subject[0, 'D', 'V'] = 2
+      subject[0, 'D', 'N'].must_equal 1
+      subject[0, 'D', 'V'].must_equal 2
+    end
+  end
+
+  describe '#each' do
+    it 'iterates over an internal table' do
+      subject.each.to_a.must_equal subject.table.to_a
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+
+require 'rubygems'
+
+gem 'minitest'
+require 'minitest/autorun'
+require 'minitest/hell'
+
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'myaso'
+
+Dir[File.expand_path('../support/**/*.rb', __FILE__)].each { |f| require f }

data/spec/support/fixtures.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+
+require 'yaml'
+
+# When this file is loaded, for each fixture file, a singleton method is
+# created within the Myaso::Fixtures module with the same name as the fixture
+# file, returning the value of the fixture.
+#
+# For example, file <tt>prefixes.yml</tt>:
+#
+#   - id: 1
+#     prefix: sub
+#   - id: 2
+#     prefix: bi
+#
+# These fixtures would be made available like so:
+#
+#   Myaso::Fixtures::PREFIXES
+#   => [{"id"=>1, "prefix"=>"sub"}, {"id"=>2, "prefix"=>"bi"}]
+#
+# You can find out all available fixtures by calling
+#
+#   Myaso::Fixtures.constants
+#   => [ :BUCKETS ]
+#
+module Myaso::Fixtures
+end
+
+fixtures_path = File.expand_path('../../fixtures', __FILE__)
+
+Dir[File.expand_path('*.yml', fixtures_path)].each do |filename|
+  const_name = File.basename(filename, '.*').upcase
+  Myaso::Fixtures.const_set(const_name, YAML.load_file(filename))
+end

data/spec/support/invoker.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+
+require 'open3'
+
+# http://dota2.ru/guides/880-invokirkhakha-sanstrajk-ni-azhydal-da/
+#
+class MiniTest::Test
+  # Quas Wex Exort.
+  #
+  def invoke_cache
+    @invoke_cache ||= {}
+  end
+
+  # So begins a new age of knowledge.
+  #
+  def invoke(*argv)
+    return invoke_cache[argv] if invoke_cache.has_key? argv
+
+    arguments = argv.dup
+    options = (arguments.last.is_a? Hash) ? arguments.pop : {}
+    executable = File.expand_path('../../../bin/myaso', __FILE__)
+
+    Open3.popen3(executable, *arguments) do |i, o, *_|
+      i.puts options[:stdin] if options[:stdin]
+      i.close
+      invoke_cache[argv] = o.readlines.map(&:chomp!)
+    end
+  end
+end

data/spec/tagger_spec.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+
+require_relative 'spec_helper'
+
+describe Myaso::Tagger do
+  let(:ngrams) { File.expand_path('../data/test.123', __FILE__) }
+  let(:lexicon) { File.expand_path('../data/test.lex', __FILE__) }
+  let(:model) { Myaso::Tagger::TnT.new(ngrams, lexicon) }
+
+  subject { Myaso::Tagger.new(model) }
+
+  describe 'annotate(sentence)' do
+    it 'should annotate one word sentences' do
+      subject.annotate(%w(братишка)).must_equal %w(e)
+    end
+
+    it 'should annotate sentences with tags' do
+      subject.annotate(%w(братишка я тебе покушать принес)).
+        must_equal(%w(a b b d e))
+    end
+
+    it 'should handle unknown words' do
+      subject.annotate(%w(мир прекрасен , как никогда)).
+        must_equal(%w(d d d d d))
+    end
+  end
+end

data/spec/tagger_tnt_spec.rb

@@ -0,0 +1,73 @@
+# encoding: utf-8
+
+require_relative 'spec_helper'
+
+describe Myaso::Tagger::Model do
+  let(:ngrams) { File.expand_path('../data/test.123', __FILE__) }
+  let(:lexicon) { File.expand_path('../data/test.lex', __FILE__) }
+
+  subject { Myaso::Tagger::TnT.new(ngrams, lexicon) }
+
+  describe '#q(t1,t2,t3)' do
+    it 'counts the quotient between trigram and bigram counts othewise' do
+      subject.q('a', 'a', 'a').must_be_close_to 0.224, 0.001
+      subject.q('b', 'a', 'b').must_be_close_to 0.287, 0.001
+    end
+  end
+
+  describe '#e(w|t)' do
+    it 'returns 0 if there is no such bunch word => tag' do
+      subject.e('братишка', 'b').must_equal(0)
+      subject.e('проголодался', 'c').must_equal(0)
+    end
+
+    it 'counts the quotient between count(word, tag) and ngrams(tag)' do
+      subject.e('братишка', 'a').must_equal(1 / 26.0)
+      subject.e('принес', 'e').must_equal(2 / 6.0)
+    end
+  end
+
+  describe '#learn!' do
+    it 'should has the same ngrams as in the gold standard' do
+      subject.ngrams.must_equal Myaso::Fixtures::NGRAMS
+    end
+
+    it 'should has the same lexicon as in the gold standard' do
+      subject.lexicon.must_equal Myaso::Fixtures::LEXICON
+    end
+
+    it 'should has the same interpolations as in the gold standard' do
+      subject.interpolations.must_equal Myaso::Fixtures::INTERPOLATIONS
+    end
+  end
+
+  describe '#start_symbol' do
+    it 'should be SENT' do
+      subject.start_symbol.must_equal 'SENT'
+    end
+  end
+
+  describe '#stop_symbol' do
+    it 'should be SENT' do
+      subject.stop_symbol.must_equal 'SENT'
+    end
+  end
+
+  describe '#conditional' do
+    it 'should compute p(0|0) as 0' do
+      subject.conditional(0, 0).must_equal 0.0
+    end
+
+    it 'should compute p(1|0) as 0' do
+      subject.conditional(1, 0).must_equal 0.0
+    end
+
+    it 'should compute p(0|1) as 0' do
+      subject.conditional(0, 1).must_equal 0.0
+    end
+
+    it 'should compute p(3|2) as 1.5' do
+      subject.conditional(3, 2).must_equal 1.5
+    end
+  end
+end