myaso 0.4.0

data/lib/myaso/tagger/tnt.rb

@@ -0,0 +1,183 @@
+# encoding: utf-8
+
+# A Tagger model that can work with TnT data files.
+#
+class Myaso::Tagger::TnT < Myaso::Tagger::Model
+  # A start tag for a sentence.
+  #
+  START = 'SENT'
+
+  # A stop tag for a sentence.
+  #
+  STOP = 'SENT'
+
+  # Unknown tag for token.
+  #
+  MISSING = '-'
+
+  # Tokens consisting of a sequence of decimal digits.
+  #
+  CARD = '@CARD'
+
+  # Decimal digits followed by punctuation.
+  #
+  CARDPUNCT = '@CARDPUNCT'
+
+  # Decimal digits followed by any suffix.
+  #
+  CARDSUFFIX = '@CARDSUFFIX'
+
+  # Decimal digits separated by dots, dashes, etc.
+  #
+  CARDSEPS = '@CARDSEPS'
+
+  # Tag frequencies to handle unknown words.
+  #
+  UNKNOWN = '@UNKNOWN'
+
+  attr_reader :ngrams_path, :lexicon_path
+
+  # The tagging model is initialized by two data files. The first one is a
+  # n-grams file that stores statistics for unigrams, bigrams, trigrams.
+  # The second one is a lexicon file that stores words and their
+  # frequencies in the source corpus.
+  #
+  # Please note that the learning stage is not so optimized, so the
+  # initialization procedure may take about 120 seconds.
+  #
+  def initialize(ngrams_path, lexicon_path, interpolations = nil)
+    @ngrams_path = File.expand_path(ngrams_path)
+    @lexicon_path = File.expand_path(lexicon_path)
+    super(interpolations)
+  end
+
+  # If word is rare, it can be one of the following categories:
+  # includes numbers, numbers and punctuation symbols, non-numbers
+  # following numbers and unknown. Otherwise, word has it's own category.
+  #
+  def classify(word)
+    return word unless rare? word
+    case word
+    when /^\d+$/ then CARD
+    when /^\d+[.,;:]+$/ then CARDPUNCT
+    when /^\d+\D+$/ then CARDSUFFIX
+    when /^\d+[.,;:\-]+(\d+[.,;:\-]+)*\d+$/ then CARDSEPS
+    else UNKNOWN
+    end
+  end
+
+  # Tagger requires the sentence start symbol to be defined.
+  #
+  def start_symbol
+    START
+  end
+
+  # Tagger requires the sentence stop symbol to be defined.
+  #
+  def stop_symbol
+    STOP
+  end
+
+  # Parse n-grams and lexicon files, and compute statistics over them.
+  #
+  def learn!
+    parse_ngrams!
+    parse_lexicon!
+    compute_interpolations! if interpolations.nil?
+  end
+
+  # Parse the n-grams file.
+  #
+  def parse_ngrams!
+    unigram, bigram = nil, nil
+
+    read(ngrams_path) do |values|
+      values[0] = unigram unless values[0]
+      values[1] = bigram unless values[1]
+
+      if values[0] && values[1] && values[2] && values[3] # a trigram
+        ngrams[*values[0..2]] = values[3].to_i
+      elsif values[0] && values[1] && values[2] && !values[3] # a bigram
+        ngrams[*values[0..1]] = values[2].to_i
+      elsif values[0] && values[1] && !values[2] && !values[3] # an unigram
+        ngrams[values[0]] = values[1].to_i
+      else
+        raise 'dafuq i just read: %s' % values.inspect
+      end
+
+      unigram, bigram = values[0], values[1]
+    end
+  end
+
+  # Parse the lexicon file.
+  #
+  def parse_lexicon!
+    read(lexicon_path) do |values|
+      values.compact!
+
+      word, word_count, rare = values.shift, values.shift.to_i, false
+      word = classify(word) if rare = (word_count == 1)
+
+      lexicon[word] += word_count
+
+      values.each_slice(2) do |tag, count|
+        lexicon[word, tag] += count.to_i
+      end
+    end
+  end
+
+  # Count coefficients for linear interpolation for evaluating
+  # q(first, second, third).
+  #
+  def compute_interpolations!
+    lambdas = [0.0, 0.0, 0.0]
+
+    unigram, bigram = nil, nil
+
+    read(ngrams_path) do |first, second, third, count|
+      first = unigram unless first
+      second = bigram unless second
+
+      unless third && count
+        unigram, bigram = first, second
+        next
+      end
+
+      count = count.to_i
+
+      f = Array.new
+
+      f << conditional(ngrams[third] - 1, ngrams.unigrams_count - 1)
+      f << conditional(ngrams[second, third] - 1, ngrams[second] - 1)
+      f << conditional(count - 1, ngrams[first, second] - 1)
+
+      index = f.index(f.max)
+
+      lambdas[index] += count if index
+    end
+
+    total = lambdas.inject(&:+)
+    @interpolations = lambdas.map! { |l| l / total }
+  end
+
+  # @private
+  #
+  def inspect
+    sprintf('#<%s @ngrams_path=%s @lexicon_path=%s @interpolations=%s>',
+      self.class.name, ngrams_path.inspect, lexicon_path.inspect,
+      interpolations.inspect)
+  end
+
+  private
+  # Read the TnT data file.
+  #
+  def read(path)
+    File.open(path) do |f|
+      until f.eof?
+        line = f.gets.chomp
+        next if line.empty? || line =~ /^%%/
+        yield line.split(/\t/).map! { |s| s.empty? ? nil : s }
+      end
+    end
+  end
+end

data/lib/myaso/version.rb

@@ -0,0 +1,9 @@
+# encoding: utf-8
+
+# Myaso is a morphological analysis and synthesis library in Ruby.
+#
+module Myaso
+  # Myaso version string.
+  #
+  VERSION = '0.4.0'
+end

data/myaso.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'myaso/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'myaso'
+  spec.version       = Myaso::VERSION
+  spec.authors       = ['Dmitry Ustalov', 'Sergey Smagin']
+  spec.email         = ['dmitry.ustalov@gmail.com']
+  spec.description   = 'Myaso is a morphological analysis library in Ruby.'
+  spec.summary       = 'Myaso is a morphological analysis and synthesis ' \
+                       'library in Ruby.'
+  spec.homepage      = 'https://github.com/dustalov/myaso'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'ffi', '~> 1.9.0'
+  spec.add_dependency 'myasorubka', '~> 0.2.0'
+  spec.add_development_dependency 'minitest', '~> 5.0'
+end

data/spec/bin_spec.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+
+require_relative 'spec_helper'
+
+describe 'CLI' do
+  it 'should print help when ran without arguments' do
+    invoke('').must_include 'Commands:'
+    invoke('').must_include 'Options:'
+  end
+
+  it 'should print version information' do
+    invoke('-v').must_include 'Myaso v%s' % Myaso::VERSION
+    invoke('--version').must_equal invoke('-v')
+  end
+
+  it 'should print help' do
+    invoke('-h').must_include 'Commands:'
+    invoke('-h').must_include 'Options:'
+    invoke('--help').must_equal invoke('-h')
+  end
+
+  it 'should evaluate parameters' do
+    invoke('-e', 'puts :hi; exit').must_include 'hi'
+    invoke('--eval', 'puts :hi; exit').must_equal(
+      invoke('-e', 'puts :hi; exit'))
+  end
+
+  it 'should consider the ngrams parameter' do
+    invoke('-n', 'test', '-e', 'puts options.ngrams; exit').
+      must_include 'test'
+    invoke('--ngrams', 'test', '-e', 'puts options.ngrams; exit').
+      must_equal(invoke('-n', 'test', '-e', 'puts options.ngrams; exit'))
+  end
+
+  it 'should consider the lexicon parameter' do
+    invoke('-l', 'test', '-e', 'puts options.lexicon; exit').
+      must_include 'test'
+    invoke('--lexicon', 'test', '-e', 'puts options.lexicon; exit').
+      must_equal(invoke('-l', 'test', '-e', 'puts options.lexicon; exit'))
+  end
+
+  it 'should annotate sentences' do
+    stdout = invoke('-n', 'spec/data/test.123', '-l', 'spec/data/test.lex',
+      'tagger', stdin: "братишка\nя\nтебе\nпокушать\nпринес")
+    stdout.must_equal ["братишка\ta", "я\tb", "тебе\tb", "покушать\td",
+      "принес\te"]
+  end
+end

data/spec/data/test.123

@@ -0,0 +1,77 @@
+%% Test set of ngrams.
+%%
+%% Possible tags are: a, b, c, d, e.
+a	26
+	a	6
+		a	1
+		b	2
+		c	1
+		SENT	1
+		e	1
+	b	8
+		c	3
+		d	3
+		e	2
+	c	4
+		a	2
+		e	1
+		SENT	1
+	d	4
+		a	3
+		e	1
+	e	3
+		b	1
+		d	1
+		SENT	1
+	SENT	3
+		a	2
+		e	1
+b	12
+	a	6
+		a	1
+		b	2
+		c	1
+		SENT	1
+		e	1
+	c	4
+		a	2
+		e	2
+	d	2
+		a	1
+		e	1
+c	10
+	a	8
+		a	1
+		b	3
+		c	2
+		d	1
+		e	1
+	e	2
+		b	1
+		d	1
+d	16
+	a	12
+		a	1
+		b	4
+		c	3
+		d	2
+		e	2
+	e	4
+		SENT	2
+		d	2
+e	6
+	d	2
+		a	1
+		e	1
+	SENT	4
+		a	1
+		c	2
+		d	1
+SENT	10
+	SENT	2
+		a	2
+	a	4
+		b	3
+		d	1
+	e	1
+		SENT	1

data/spec/data/test.lex

@@ -0,0 +1,10 @@
+%% Test	lexicon. It should be crafty.
+%%
+%% Tags are: a, b, c, d, e.
+%%
+братишка		2	a	1	e	1
+я		6	b	4	c	2
+тебе	11	b	8	c	2	a	1
+покушать	5	d	5
+принес	7	d	5	e	2
+проголодался	1	d	1

data/spec/fixtures/interpolations.yml

@@ -0,0 +1,4 @@
+---
+- 0.189873417721519
+- 0.4177215189873418
+- 0.3924050632911392

data/spec/fixtures/lexicon.yml

@@ -0,0 +1,32 @@
+--- !ruby/object:Myaso::Lexicon
+table:
+  бра:
+    братишка:
+      ! '': 2
+      a: 1
+      e: 1
+  я:
+    я:
+      ! '': 6
+      b: 4
+      c: 2
+  теб:
+    тебе:
+      ! '': 11
+      b: 8
+      c: 2
+      a: 1
+  пок:
+    покушать:
+      ! '': 5
+      d: 5
+  при:
+    принес:
+      ! '': 7
+      d: 5
+      e: 2
+  '@UN':
+    '@UNKNOWN':
+      ! '': 1
+      d: 1
+tags: 

data/spec/fixtures/ngrams.yml

@@ -0,0 +1,106 @@
+--- !ruby/object:Myaso::Ngrams
+table:
+  a:
+    ! '':
+      ! '': 26
+    a:
+      ! '': 6
+      a: 1
+      b: 2
+      c: 1
+      SENT: 1
+      e: 1
+    b:
+      ! '': 8
+      c: 3
+      d: 3
+      e: 2
+    c:
+      ! '': 4
+      a: 2
+      e: 1
+      SENT: 1
+    d:
+      ! '': 4
+      a: 3
+      e: 1
+    e:
+      ! '': 3
+      b: 1
+      d: 1
+      SENT: 1
+    SENT:
+      ! '': 3
+      a: 2
+      e: 1
+  b:
+    ! '':
+      ! '': 12
+    a:
+      ! '': 6
+      a: 1
+      b: 2
+      c: 1
+      SENT: 1
+      e: 1
+    c:
+      ! '': 4
+      a: 2
+      e: 2
+    d:
+      ! '': 2
+      a: 1
+      e: 1
+  c:
+    ! '':
+      ! '': 10
+    a:
+      ! '': 8
+      a: 1
+      b: 3
+      c: 2
+      d: 1
+      e: 1
+    e:
+      ! '': 2
+      b: 1
+      d: 1
+  d:
+    ! '':
+      ! '': 16
+    a:
+      ! '': 12
+      a: 1
+      b: 4
+      c: 3
+      d: 2
+      e: 2
+    e:
+      ! '': 4
+      SENT: 2
+      d: 2
+  e:
+    ! '':
+      ! '': 6
+    d:
+      ! '': 2
+      a: 1
+      e: 1
+    SENT:
+      ! '': 4
+      a: 1
+      c: 2
+      d: 1
+  SENT:
+    ! '':
+      ! '': 10
+    SENT:
+      ! '': 2
+      a: 2
+    a:
+      ! '': 4
+      b: 3
+      d: 1
+    e:
+      ! '': 1
+      SENT: 1