myasorubka 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f19b580bd580d01ac69b901c274b066f936dfd39
+  data.tar.gz: 1d5dde45cbdb11897a595b0dcd42dd2dbd096980
+SHA512:
+  metadata.gz: a330c97844b4860cfbdfd4ad5bcc950d67bdbb4ef02489b12afe7e84c16167a384ea6523c05e315bec4007b6070351a331d2d014cf8b2296310a012a11977840
+  data.tar.gz: bb98c6a3055464bea25b6ad27a20c66abc02e3c9847d71c3cde993b8a1c9673826633909608bdc81eba25c4645943a416c32f1607382e071a97501da1f398b03

data/.gitignore

@@ -0,0 +1,45 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## RUBINIUS
+*.rbc
+
+## NETBEANS
+nbproject
+
+## REDCAR
+.redcar
+
+## RVM
+.rvmrc
+.ruby-version
+.ruby-gemset
+
+## RUBINIUS
+.rbx
+
+## BUNDLER
+.bundle
+Gemfile.lock
+
+## YARD
+.yardoc
+doc
+
+## PROJECT::GENERAL
+coverage
+pkg
+
+## PROJECT::SPECIFIC

data/.travis.yml

@@ -0,0 +1,7 @@
+branches:
+  only:
+    - master
+rvm:
+  - 2.0.0
+  - jruby-19mode
+  - rbx-19mode

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2011-2013 Dmitry Ustalov
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,23 @@
+Myasorubka
+==========
+
+Myasorubka is a morphological data processor that supports
+[AOT](http://aot.ru) and [MULTEXT-East](http://nl.ijs.si/ME/)
+notations.
+
+## Contributing
+
+1. Fork it;
+2. Create your feature branch (`git checkout -b my-new-feature`);
+3. Commit your changes (`git commit -am 'Added some feature'`);
+4. Push to the branch (`git push origin my-new-feature`);
+5. Create new Pull Request.
+
+## Build Status [<img src="https://secure.travis-ci.org/ustalov/myasorubka.png"/>](http://travis-ci.org/ustalov/myasorubka)
+
+## Dependency Status [<img src="https://gemnasium.com/ustalov/myasorubka.png"/>](https://gemnasium.com/ustalov/myasorubka)
+
+## Copyright
+Copyright (c) 2011-2013 [Dmitry Ustalov]. See LICENSE for details.
+
+[Dmitry Ustalov]: http://eveel.ru

data/Rakefile

@@ -0,0 +1,11 @@
+# encoding: utf-8
+
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+Rake::TestTask.new do |test|
+  test.pattern = 'spec/**/*_spec.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/aot-russian

@@ -0,0 +1,7 @@
+#!/bin/sh
+rake clean aot \
+  mrd=morphs.mrd \
+  tab=rgramtab.tab \
+  encoding=CP1251 \
+  language=russian \
+  --trace

data/lib/myasorubka.rb

@@ -0,0 +1,4 @@
+# encoding: utf-8
+
+require 'myasorubka/version'
+require 'myasorubka/msd'

data/lib/myasorubka/aot.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+
+# http://aot.ru/
+module Myasorubka::AOT
+  require 'myasorubka/aot/dictionary'
+  require 'myasorubka/aot/gramtab'
+  require 'myasorubka/aot/tags'
+end

data/lib/myasorubka/aot/dictionary.rb

@@ -0,0 +1,125 @@
+# encoding: utf-8
+
+# MRD file is a text file that contains a morphological dictionary of
+# a natural language. MRD is an abbreviation of "morphological dictionary".
+#
+# All words in MRD file are written in UPPERCASE. One MRD file has the
+# following sections: section of flexion and prefix models, section of
+# accentual models, section of user sessions, session of prefix sets,
+# section of lemmas.
+#
+class Myasorubka::AOT::Dictionary
+  attr_reader :lines, :language
+  attr_reader :rules_offset, :accents_offset, :logs_offset,
+              :prefixes_offset, :lemmas_offset
+
+  # The parser should be initialized by passing filename and language
+  # parameters.
+  #
+  def initialize(filename, language = nil, ee = nil, ie = Encoding.default_external)
+    encoding = { internal_encoding: ie, external_encoding: ee }
+    @lines, @language = File.readlines(filename, $/, encoding), language
+
+    @rules_offset = 0
+    @accents_offset = rules_offset + rules.length + 1
+    @logs_offset = accents_offset + accents.length + 1
+    @prefixes_offset = logs_offset + logs.length + 1
+    @lemmas_offset = prefixes_offset + prefixes.length + 1
+  end
+
+  # MRD file section handling helper class.
+  #
+  # Each section is a set of records, one per line. The number of
+  # all records of the section is written in the very beginning of
+  # the section ata separate line.
+  #
+  class Section
+    include Enumerable
+
+    attr_reader :lines, :offset, :length, :parser
+
+    # :nodoc:
+    def initialize(lines, offset, &block)
+      @lines, @offset = lines, offset
+      @length = lines[offset].strip.to_i
+      @parser = block || proc { |s| s }
+    end
+
+    # :nodoc:
+    def [] id
+      if id < 0 or id >= length
+        raise ArgumentError,
+          'invalid id=%d when offset=%d and length=%d' %
+          [id, offset, length]
+      end
+
+      parser.call(lines[offset + 1 + id].strip)
+    end
+
+    # :nodoc:
+    def each(&block)
+      length.times { |id| block.call self[id] }
+    end
+  end
+
+  # Rules section accessor.
+  #
+  def rules
+    @rules ||= Section.new(lines, rules_offset) do |line|
+      line.split('%').map do |rule_line|
+        next unless rule_line && !rule_line.empty?
+
+        suffix, ancode, prefix = rule_line.split '*'
+
+        case language
+        when :russian then
+          suffix &&= suffix.tr 'Ёё', 'Ее'
+          prefix &&= prefix.tr 'Ёё', 'Ее'
+        end
+
+        [suffix, ancode[0..1], prefix]
+      end.compact
+    end
+  end
+
+  # Accents section accessor.
+  #
+  def accents
+    @accents ||= Section.new(lines, accents_offset)
+  end
+
+  # Logs section accessor.
+  #
+  def logs
+    @logs ||= Section.new(lines, logs_offset)
+  end
+
+  # Prefixes section accessor.
+  #
+  def prefixes
+    @prefixes ||= Section.new(lines, prefixes_offset)
+  end
+
+  # Lemmas section accessor.
+  #
+  def lemmas
+    @lemmas ||= Section.new(lines, lemmas_offset) do |line|
+      stem, rule_id, accent_id, session_id, ancode, prefix_id = line.split
+
+      case language
+      when :russian then
+        stem &&= stem.tr 'Ёё', 'Ее'
+      end
+
+      Array.new.tap do |result|
+        result <<
+          (stem == '#' ? nil : stem) <<
+          rule_id.to_i <<
+          accent_id.to_i <<
+          session_id.to_i <<
+          (ancode == '-' ? nil : ancode[0..1]) <<
+          (prefix_id == '-' ? nil : prefix_id.to_i)
+      end
+    end
+  end
+end

data/lib/myasorubka/aot/gramtab.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+
+# Tab file contains all possible full morphological  patterns for the
+# words.
+#
+# One line in a Tab file looks like as follows:
+#
+#   <ancode> <useless_number> <pos> <grammemes>
+#
+# An ancode is an ID, which consists of two letters and which
+# uniquely identifies a morphological pattern. A morphological pattern
+# consists of:
+#
+#   <pos> and <grammemes>
+#
+# A MRD file refers to a Tab file, which is language-dependent.
+#
+class Myasorubka::AOT::Gramtab
+  attr_reader :ancodes, :language
+
+  # :nodoc:
+  def initialize(filename, ee = nil, ie = Encoding.default_external)
+    @ancodes, @language, id = {}, language, -1
+    encoding = { internal_encoding: ie, external_encoding: ee }
+
+    File.readlines(filename, $/, encoding).each do |line, i|
+      next if line.empty? or line.start_with?('//')
+      ancode, _, pos, grammemes = line.split
+      ancodes[ancode] = { id: id += 1, pos: pos, grammemes: grammemes || '' }
+    end
+  end
+end

data/lib/myasorubka/aot/tags.rb

@@ -0,0 +1,326 @@
+# encoding: utf-8
+
+# AOT-to-MSD morphosyntactic descriptors translator.
+#
+class Myasorubka::AOT::Tags
+  def self.join(hash, msd)
+    msd.grammemes.inject(hash) { |h, (k, v)| h[k.to_s] = v.to_s; h }
+  end
+
+  # Russian language helpers.
+  #
+  module Russian
+    def self.gender(msd, grammemes)
+      msd[:gender] = if grammemes.include? 'мр'
+        :masculine
+      elsif grammemes.include? 'жр'
+        :feminine
+      elsif grammemes.include? 'ср'
+        :neuter
+      elsif grammemes.include? 'мр-жр'
+        :common
+      end
+      msd
+    end
+
+    def self.animate(msd, grammemes)
+      msd[:animate] = if grammemes.include? 'од'
+        :yes
+      elsif grammemes.include? 'но'
+        :no
+      end
+      msd
+    end
+
+    def self.number(msd, grammemes)
+      msd[:number] = if grammemes.include? 'ед'
+        :singular
+      elsif grammemes.include? 'мн'
+        :plural
+      end
+      msd
+    end
+
+    def self.case(msd, grammemes)
+      msd[:case] = if grammemes.include? 'им'
+        :nominative
+      elsif grammemes.include? 'рд'
+        :genitive
+      elsif grammemes.include? 'дт'
+        :dative
+      elsif grammemes.include? 'вн'
+        :accusative
+      elsif grammemes.include? 'тв'
+        :instrumental
+      elsif grammemes.include? 'пр'
+        :locative
+      elsif grammemes.include? 'зв'
+        :vocative
+      end
+      msd
+    end
+
+    def self.case2(msd, grammemes)
+      if grammemes.include? '2'
+        msd[:case2] = :partitive if :genitive == msd[:case]
+        msd[:case2] = :locative if :locative == msd[:case]
+      end
+      msd
+    end
+
+    def self.aspect(msd, grammemes)
+      msd[:aspect] = if grammemes.include? 'св'
+        :perfective
+      elsif grammemes.include? 'нс'
+        :progressive
+      end
+      msd
+    end
+
+    def self.voice(msd, grammemes)
+      msd[:voice] = if grammemes.include? 'дст'
+        :active
+      elsif grammemes.include? 'стр'
+        :passive
+      elsif :verb == msd[:pos]
+        :medial
+      end
+      msd
+    end
+
+    def self.tense(msd, grammemes)
+      msd[:tense] = if grammemes.include? 'нст'
+        :present
+      elsif grammemes.include? 'прш'
+        :past
+      elsif grammemes.include? 'буд'
+        :future
+      end
+      msd
+    end
+
+    def self.person(msd, grammemes)
+      msd[:person] = if grammemes.include? '1л'
+        :first
+      elsif grammemes.include? '2л'
+        :second
+      elsif grammemes.include? '3л'
+        :third
+      elsif grammemes.include? 'безл'
+        nil
+      end
+      msd
+    end
+
+    def self.definiteness(msd, grammemes)
+      msd[:definiteness] = if grammemes.include? 'кр'
+        :short_art
+      end
+      msd
+    end
+
+    def self.degree(msd, grammemes)
+      msd[:degree] = if grammemes.include? 'сравн'
+        :comparative
+      elsif grammemes.include? 'прев'
+        :superlative
+      end
+      msd
+    end
+  end
+
+  # Russian language.
+  #
+  def self.russian(pos_line, grammemes_line)
+    grammemes = grammemes_line.split(',').map do |grammeme|
+      UnicodeUtils.downcase(grammeme)
+    end
+
+    msd = Myasorubka::MSD.new(Myasorubka::MSD::Russian)
+
+    if grammemes.include? 'aббр'
+      msd[:pos] = :abbreviation
+      msd[:syntactic_type] = if 'Н' == pos_line
+        :adverbial
+      else
+        :nominal
+      end
+      pos_line = 'АББР'
+    end
+
+    case UnicodeUtils.upcase(pos_line)
+    when 'С' then begin
+      msd[:pos] = :noun
+      msd[:type] = if (grammemes & [ 'имя', 'фам', 'отч', 'жарг', 'арх', 'проф', 'опч' ]).empty?
+        :common
+      else
+        :proper
+      end
+      [ :gender, :number, :case, :animate, :case2 ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    when 'П' then begin
+      msd[:pos] = :adjective
+      msd[:type] = if grammemes.include? 'кач'
+        :qualificative
+      elsif grammemes.include? 'притяж'
+        :possessive
+      end
+      msd[:degree] = :positive
+      msd[:definiteness] = :full_art
+      [ :degree, :gender, :number, :case, :definiteness ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    when 'МС' then begin
+      msd[:pos] = :pronoun
+      msd[:type] = :possessive if grammemes.include? 'притяж'
+      [ :person, :gender, :number, :case, :animate ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    when 'МС-ПРЕДК' then begin
+      msd[:pos] = :pronoun
+      msd[:type] = :possessive if grammemes.include? 'притяж'
+      msd[:case] = :genitive
+      [ :person, :gender, :number, :animate ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    when 'МС-П' then begin
+      msd[:pos] = :pronoun
+      msd[:type] = :possessive if grammemes.include? 'притяж'
+      msd[:syntactic_type] = :adjectival
+      [ :person, :gender, :number, :case, :animate ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    when 'Г' then begin
+      msd[:pos] = :verb
+      msd[:type] = :main
+      msd[:vform] = if grammemes.include? 'пвл'
+        :imperative 
+      end
+      msd[:definiteness] = :full_art
+      [ :tense, :person, :number, :gender, :voice, :definiteness,
+        :aspect, :case ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    when 'ПРИЧАСТИЕ' then begin
+      msd[:pos] = :verb
+      msd[:vform] = :participle
+      msd[:definiteness] = :full_art
+      [ :tense, :person, :number, :gender, :voice, :definiteness,
+        :aspect, :case ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    when 'ДЕЕПРИЧАСТИЕ' then begin
+      msd[:pos] = :verb
+      msd[:vform] = :gerund
+      msd[:definiteness] = :full_art
+      [ :tense, :person, :number, :gender, :voice, :definiteness,
+        :aspect, :case ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    when 'ИНФИНИТИВ' then begin
+      msd[:pos] = :verb
+      msd[:vform] = :infinitive
+      msd[:definiteness] = :full_art
+      [ :tense, :person, :number, :gender, :voice, :definiteness,
+        :aspect, :case ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    when 'ЧИСЛ' then begin
+      msd[:pos] = :numeral
+      msd[:type] = :cardinal
+      [ :gender, :number, :case, :animate ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    when 'ЧИСЛ-П' then begin
+      msd[:pos] = :numeral
+      msd[:type] = :ordinal
+      [ :gender, :number, :case, :animate ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    when 'Н' then begin
+      msd[:pos] = :adverb
+      msd[:degree] = :positive
+      [ :degree ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    when 'ПРЕДК' then begin
+      msd[:pos] = :adverb
+      msd[:degree] = :positive
+      [ :degree ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    when 'ПРЕДЛ' then begin
+      msd[:pos] = :adposition
+      msd[:type] = :preposition
+      [ :case ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    when 'СОЮЗ' then begin
+      msd[:pos] = :conjunction
+    end
+    when 'МЕЖД' then begin
+      msd[:pos] = :interjection
+    end
+    when 'ЧАСТ' then begin
+      msd[:pos] = :particle
+    end
+    when 'ВВОДН' then begin
+      msd[:pos] = :adposition
+    end
+    when 'КР_ПРИЛ' then begin
+      msd[:pos] = :adjective
+      msd[:type] = if grammemes.include? 'кач'
+        :qualificative
+      elsif grammemes.include? 'притяж'
+        :possessive
+      end
+      msd[:degree] = :positive
+      [ :degree, :gender, :number, :case ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+      msd[:definiteness] = :short_art
+    end
+    when 'КР_ПРИЧАСТИЕ' then begin
+      msd[:pos] = :verb
+      msd[:vform] = :participle
+      [ :tense, :person, :number, :gender, :voice,
+        :aspect, :case ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+      msd[:definiteness] = :short_art
+    end
+    when 'АББР' then begin
+      [ :gender, :number, :case ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    when '*' then begin
+      msd[:pos] = :crutch
+      [ :gender, :animate, :number, :case, :case2, :aspect,
+        :voice, :tense, :person, :definiteness,
+        :degree ].each do |attribute|
+        Russian.send(attribute, msd, grammemes)
+      end
+    end
+    else
+      msd[:pos] = :residual
+    end
+
+    msd
+  end
+end