yandex_mystem 0.0.2 → 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cba5e52c1b005179967ad5a066786c5050342018
+  data.tar.gz: f9112c1e6ce51124d48279f8218fa5d522b2ded8
+SHA512:
+  metadata.gz: 9337422e5f0293516d4b8c48a5adc3ae1d48f1682d403d700480a4f9f71ee5a71f68bec89525001b6dc05ce7e01193a2bc609c089e8e603fa2bff271740d2a20
+  data.tar.gz: 1c84c101387557db4f8760445168412b3f14053e678689c63e2237b21137b978dea4c23f4633f26e2cd33a07cfe8b061f90339764056072d59a1bc58dd3de0a8

data/.rvmrc

@@ -1 +1 @@
-rvm use 1.9.3 --create
+rvm use 2.0.0 --create

data/.travis.yml

@@ -0,0 +1,3 @@
+rvm:
+  - 1.9.3
+  - 2.0.0

data/README.md

@@ -1,4 +1,7 @@
-# Yandex Mystem
+# Yandex Mystem 0.1.0
+
+[![Build Status](https://secure.travis-ci.org/dmitry/yandex_mystem.png?branch=master)](http://travis-ci.org/dmitry/yandex_mystem) [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/dmitry/yandex_mystem/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
+
 
 ## Introduction
 
@@ -27,4 +30,5 @@ This gem contains executables for there platforms:
 
 ## Usage
 
-    YandexMystem::Base.stem 'О предложении в котором много слов.'
+    YandexMystem::Simple.stem 'О предложении в котором много слов.'
+    YandexMystem::Extended.stem 'нет сов'

data/lib/yandex_mystem/version.rb

@@ -1,3 +1,3 @@
 module YandexMystem
-  VERSION = "0.0.2"
+  VERSION = '0.1.0'
 end

data/lib/yandex_mystem.rb

@@ -3,35 +3,25 @@ require 'yandex_mystem/version'
 
 module YandexMystem
   class Base
-    # TODO add -i
+    WORD_SCANNER_REGEXP = /^([^\{]+)\{(.+)\}$/.freeze
+
     def self.stem(text)
-      exec = Array(command).tap do |c|
-        c << '-e utf-8 -n'
-      end.join(' ')
+      exec = [command, self::ARGUMENTS].join(' ')
 
-      data = Open3.popen3(exec) do |stdin, stdout, stderr|
+      data = Open3.popen3(exec) do |stdin, stdout, _|
         stdin.write text
         stdin.close
-        #stderr.read
         stdout.read
       end
 
-      data = data.scan(/^([^\{]+)\{(.+)\}$/).map do |(word, words)|
-        words = words.split('|').select do |w|
-          !(w =~ /.+\?\?$/)
-        end
-
-        [word, words]
-      end.flatten(1)
-
-      Hash[*data]
+      parse(data)
     end
 
-    private
-
     def self.command
-      path = Pathname.new(__FILE__) + '../../app/'
-      path + "mystem-#{command_postfix}"
+      @command ||= begin
+        path = Pathname.new(__FILE__) + '../../app/'
+        path + "mystem-#{command_postfix}"
+      end
     end
 
     def self.command_postfix
@@ -51,4 +41,56 @@ module YandexMystem
         end
     end
   end
+
+  class Simple < Base
+    ARGUMENTS = '-e utf-8 -n'
+
+    NOT_INCLUDE_REGEXP = /.+\?\?$/.freeze
+
+
+    def self.parse(data)
+      parsed = data.scan(WORD_SCANNER_REGEXP).map do |(word, words)|
+        words = words.split('|').select do |w|
+          !(w =~ NOT_INCLUDE_REGEXP)
+        end
+
+        [word, words]
+      end.flatten(1)
+
+      Hash[*parsed]
+    end
+  end
+
+  class Extended < Base
+    ARGUMENTS = '-e utf-8 -nifg'
+
+    REGEXP = /([^\|:]+):([0-9\.]+)=([A-Z]+)/
+
+    Word = Struct.new(:word, :frequency, :part)
+
+    def self.parse(data)
+      parsed = {}
+
+      data.scan(WORD_SCANNER_REGEXP).each do |(word, words)|
+        unless parsed.key?(word)
+          words = words.scan(REGEXP).map do |w|
+            to_word(w)
+          end
+
+          unless words.size.zero?
+            parsed[word] = words.sort_by(&:frequency).reverse
+          end
+        end
+      end
+
+      parsed
+    end
+
+    private
+
+    def self.to_word(w)
+      word, frequency, part = w
+      Word.new(word, frequency.to_f, part)
+    end
+  end
 end

data/spec/yandex_mystem_spec.rb

@@ -2,14 +2,54 @@
 require 'spec_helper'
 
 describe YandexMystem do
-  it "should stem words" do
-    data = YandexMystem::Base.stem('мальчики мальчиков девочки девочек компьютеров компьютере сов пошли elements')
-    data['мальчики'].should eq ['мальчик']
-    data['мальчиков'].should eq ['мальчик', "мальчиков", "мальчиковый"]
-    data['девочки'].should eq ['девочка']
-    data['девочек'].should eq ['девочка']
-    data['сов'].should eq ['сова']
-    data['пошли'].should eq %w(пойти посылать)
-    data['elements'].should eq []
+  context YandexMystem::Simple do
+    it "should stem words" do
+      data = YandexMystem::Simple.stem('мальчики мальчиков девочки девочек компьютеров компьютере сов пошли elements')
+      data['мальчики'].should eq %w(мальчик)
+      data['мальчиков'].should eq %w(мальчик мальчиков мальчиковый)
+      data['девочки'].should eq %w(девочка)
+      data['девочек'].should eq %w(девочка)
+      data['сов'].should eq %w(сова)
+      data['пошли'].should eq %w(пойти посылать)
+      data['elements'].should eq []
+    end
   end
-end
+
+  context YandexMystem::Extended do
+    it 'latin words' do
+      words = YandexMystem::Extended.stem('elements')
+
+      words.size.should eq 0
+
+      words['elements'].should be_nil
+    end
+
+    it 'multiple definitions' do
+      words = YandexMystem::Extended.stem('девочка мальчиков пошла к комьютерам искать, в прочем, как и всегда')
+
+      words.size.should eq 11
+
+      words['девочка'].first.word.should eq 'девочка'
+      words['девочка'].first.frequency.should eq 185.2
+      words['девочка'].first.part.should eq 'S'
+
+
+      words['мальчиков'].size.should eq 3
+    end
+
+    it 'sort by frequency' do
+      words = YandexMystem::Extended.stem('сосланный')
+
+      words['сосланный'].size.should eq 2
+
+      words['сосланный'].map(&:word).should eq %w(ссылать сосланный)
+      words['сосланный'].map(&:frequency).should eq [1.2, 0.5]
+      words['сосланный'].map(&:part).should eq %w(V S)
+    end
+
+    it 'set multiple times the same word, but lowercase and uppercase is different' do
+      YandexMystem::Extended.should_receive(:to_word).exactly(4).times.and_return(YandexMystem::Extended.const_get(:Word).new('в', 2, 'S'))
+      YandexMystem::Extended.stem('В в в в')
+    end
+  end
+end

data/yandex_mystem.gemspec

@@ -16,4 +16,5 @@ Gem::Specification.new do |gem|
   gem.version       = YandexMystem::VERSION
 
   gem.add_development_dependency "rspec", '~> 2.8'
+  gem.add_development_dependency "rake", '~> 10.1'
 end

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: yandex_mystem
 version: !ruby/object:Gem::Version
-  version: 0.0.2
-  prerelease: 
+  version: 0.1.0
 platform: ruby
 authors:
 - Dmitry Polushkin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-28 00:00:00.000000000 Z
+date: 2013-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -22,11 +20,24 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.8'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.1'
 description: Mystem is a software that provided by the Yandex only for non-commercial
   project. With use of it you can detect base forms of the words in a text, make a
   simple morphological analysis of russian words.
@@ -39,6 +50,7 @@ files:
 - .gitignore
 - .rspec
 - .rvmrc
+- .travis.yml
 - Gemfile
 - README.md
 - Rakefile
@@ -53,26 +65,25 @@ files:
 - yandex_mystem.gemspec
 homepage: ''
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Yandex Mystem makes morphological analysis of a russian text
 test_files: []