yandex_mystem 0.1.1 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a6c7e0e979e518cf69b18065ac82fef6341fa401
-  data.tar.gz: d21da1203a6bff374411f82ed2e8d044cf88e821
+  metadata.gz: d7c1c4fcbbbafd601e0457db5366a8d82639955d
+  data.tar.gz: 3da18f1e83a23747a51786fcda68b5ccd109dfb2
 SHA512:
-  metadata.gz: 79d31c75be6137c8d4fa5cc451fd4ed54f1fe76fb001460b10d4d33d8c1530e1a154f20979de116f51a5cf940a594d1829682bca9594bd61bfa9a624397a35ee
-  data.tar.gz: edb8e0a6d1cec46d4f5a5db9412a57ce9698808685e1bea2990c2a44e53497f30442bb8fd82dee8f73e71cae497cf12fab66515d4eab3b3dd099404fc22da88b
+  metadata.gz: fdca784ffa0621fc345ffa080e00131c43db0ac86f14012082271ed378a1cfa32ff1145acfecb77ef2d336e8f6a8ed4e976375b81ed3fbedbfd94aaf7213ba97
+  data.tar.gz: 6fa8de5e45b069137e46165d6da1cd7d0209b297e85de996a30340d1171cf8d85eb1194787f0fa40bae12fbb4ccd92ecef0601462cfdc3e2e6664dbf9ff8f234

data/README.md

@@ -3,6 +3,12 @@
 [![Build Status](https://secure.travis-ci.org/dmitry/yandex_mystem.png?branch=master)](http://travis-ci.org/dmitry/yandex_mystem) [![Gem Version](https://badge.fury.io/rb/yandex_mystem.png)](http://badge.fury.io/rb/yandex_mystem) [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/dmitry/yandex_mystem/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
 
 
+## Version 3.0 not compatible with previous
+
+Mystem 3.0 support JSON format, so we should use this option.
+This gem (yandex_mystem) now returns information in JSON-like ruby native format (array of hashes).
+More info about Mystem changes: http://api.yandex.ru/mystem/downloads/
+
 ## Introduction
 
 Mystem is a software that provided by the Yandex only for non-commercial project. With use of it you can detect base forms of the words in a text, make a simple morphological analysis of russian words.
@@ -13,7 +19,7 @@ Mystem is a software that provided by the Yandex only for non-commercial project
 
 ## License
 
-First of all, read license on http://company.yandex.ru/technologies/mystem/
+First of all, read license on http://api.yandex.ru/mystem/
 
 `Mystem` available only for non-commercial usage.
 
@@ -21,14 +27,14 @@ First of all, read license on http://company.yandex.ru/technologies/mystem/
 
 This gem contains executables for there platforms:
 
-* Windows
-* Linux 2.6 32-bit
-* Linux 2.6 64-bit
-* Mac OS X 10.5
+* Windows 7 32-bit
+* Linux 3.5 32-bit
+* Linux 3.1 64-bit
+* Freebsd 9.0 64-bit
 
-...of six, FreeBSD not in the gem. If you need it, add pull request or issue.
+Note: Mystem 3.0 not support Mac OS X, sorry.
 
 ## Usage
 
     YandexMystem::Simple.stem 'О предложении в котором много слов.'
-    YandexMystem::Extended.stem 'нет сов'
+    YandexMystem::Raw.stem 'нет сов'

data/lib/yandex_mystem/version.rb

@@ -1,3 +1,3 @@
 module YandexMystem
-  VERSION = '0.1.1'
+  VERSION = '3.0.0'
 end

data/lib/yandex_mystem.rb

@@ -1,10 +1,12 @@
+# encoding: utf-8
+
 require 'open3'
 require 'yandex_mystem/version'
 require 'pathname'
+require 'json'
 
 module YandexMystem
-  class Base
-    WORD_SCANNER_REGEXP = /^([^\{]+)\{(.+)\}$/.freeze
+  class Base    
 
     def self.stem(text)
       exec = [command, self::ARGUMENTS].join(' ')
@@ -20,7 +22,7 @@ module YandexMystem
 
     def self.command
       @command ||= begin
-        path = Pathname.new(__FILE__) + '../../app/'
+        path = Pathname.new(__FILE__) + '../../bin/'
         path + "mystem-#{command_postfix}"
       end
     end
@@ -34,9 +36,9 @@ module YandexMystem
         when /64.+linux$/
           'linux-64'
         when /darwin/
-          'mac'
+          raise 'Mystem 3.0 does not support Max OS X.'
         when /freebsd/
-          raise 'Create an issue or add pull request on a github.'
+          'bsd'
         else
           raise 'Unknown OS'
         end
@@ -44,54 +46,19 @@ module YandexMystem
   end
 
   class Simple < Base
-    ARGUMENTS = '-e utf-8 -n'
-
-    NOT_INCLUDE_REGEXP = /.+\?\?$/.freeze
+    ARGUMENTS = '-e utf-8 -n --format json'    
 
 
     def self.parse(data)
-      parsed = data.scan(WORD_SCANNER_REGEXP).map do |(word, words)|
-        words = words.split('|').select do |w|
-          !(w =~ NOT_INCLUDE_REGEXP)
-        end
-
-        [word, words]
-      end.flatten(1)
-
-      Hash[*parsed]
+      Hash[ JSON.parse('[' + data.split("\n").join(",") + ']', :symbolize_names => true).inject([]){|s, h| s + [[ h[:text], h[:analysis].map{|a| a[:lex]} ]]}  ]
     end
   end
 
-  class Extended < Base
-    ARGUMENTS = '-e utf-8 -nifg'
-
-    REGEXP = /([^\|:]+):([0-9\.]+)=([A-Z]+)/
-
-    Word = Struct.new(:word, :frequency, :part)
+  class Raw < Base
+    ARGUMENTS = '-e utf-8 -ig -n --weight --format json --eng-gr'        
 
     def self.parse(data)
-      parsed = {}
-
-      data.scan(WORD_SCANNER_REGEXP).each do |(word, words)|
-        unless parsed.key?(word)
-          words = words.scan(REGEXP).map do |w|
-            to_word(w)
-          end
-
-          unless words.size.zero?
-            parsed[word] = words.sort_by(&:frequency).reverse
-          end
-        end
-      end
-
-      parsed
-    end
-
-    private
-
-    def self.to_word(w)
-      word, frequency, part = w
-      Word.new(word, frequency.to_f, part)
-    end
+      JSON.parse('[' + data.split("\n").join(",") + ']', :symbolize_names => true)
+    end    
   end
 end

data/spec/yandex_mystem_spec.rb

@@ -4,7 +4,28 @@ require 'spec_helper'
 describe YandexMystem do
   context YandexMystem::Simple do
     it "should stem words" do
-      data = YandexMystem::Simple.stem('мальчики мальчиков девочки девочек компьютеров компьютере сов пошли elements')
+      data = YandexMystem::Simple.stem("мальчики мальчиков девочки девочек компьютеров компьютере сов пошли elements")
+      data['мальчики'].should eq %w(мальчик)
+      data['мальчиков'].should eq %w(мальчик мальчиков мальчиковый)
+      data['девочки'].should eq %w(девочка)
+      data['девочек'].should eq %w(девочка)
+      data['сов'].should eq %w(сова)
+      data['пошли'].should eq %w(пойти посылать)
+      data['elements'].should eq []
+    end
+    
+    it "should stem words in few lines" do
+      data = YandexMystem::Simple.stem(%[
+                                       
+                                       
+                                       мальчики 
+                                       мальчиков 
+                                      девочки девочек компьютеров компьютере сов пошли 
+                                      elements
+                                      
+                                      
+                                      
+                                      ])
       data['мальчики'].should eq %w(мальчик)
       data['мальчиков'].should eq %w(мальчик мальчиков мальчиковый)
       data['девочки'].should eq %w(девочка)
@@ -15,41 +36,32 @@ describe YandexMystem do
     end
   end
 
-  context YandexMystem::Extended do
+  context YandexMystem::Raw do
     it 'latin words' do
-      words = YandexMystem::Extended.stem('elements')
-
-      words.size.should eq 0
+      response = YandexMystem::Raw.stem('Elements')
 
-      words['elements'].should be_nil
+      response.size.should eq 1
+      response[0][:analysis].should eq []      
+      response[0][:text].should eq 'Elements'      
     end
 
     it 'multiple definitions' do
-      words = YandexMystem::Extended.stem('девочка мальчиков пошла к комьютерам искать, в прочем, как и всегда')
+      response = YandexMystem::Raw.stem('девочка мальчиков пошла к комьютерам искать, в прочем, как и всегда')
 
-      words.size.should eq 11
+      response.size.should eq 11
 
-      words['девочка'].first.word.should eq 'девочка'
-      words['девочка'].first.frequency.should eq 185.2
-      words['девочка'].first.part.should eq 'S'
+      response.find_all{|h| h[:text] == 'девочка'}.first[:analysis].first[:lex].should eq 'девочка'
+      response.find_all{|h| h[:text] == 'девочка'}.first[:analysis].first[:wt].should eq 1
+      response.find_all{|h| h[:text] == 'девочка'}.first[:analysis].first[:gr].should eq 'S,f,anim=nom,sg'      
 
-
-      words['мальчиков'].size.should eq 3
+      response.find_all{|h| h[:text] == 'мальчиков'}.first[:analysis].size.should eq 3
     end
 
-    it 'sort by frequency' do
-      words = YandexMystem::Extended.stem('сосланный')
-
-      words['сосланный'].size.should eq 2
-
-      words['сосланный'].map(&:word).should eq %w(ссылать сосланный)
-      words['сосланный'].map(&:frequency).should eq [1.2, 0.5]
-      words['сосланный'].map(&:part).should eq %w(V S)
+    it 'get geo name' do
+      response = YandexMystem::Raw.stem('Москва')
+      response.find_all{|h| h[:text] == 'Москва'}.first[:analysis].first[:gr].should include 'geo'
     end
 
-    it 'set multiple times the same word, but lowercase and uppercase is different' do
-      YandexMystem::Extended.should_receive(:to_word).exactly(4).times.and_return(YandexMystem::Extended.const_get(:Word).new('в', 2, 'S'))
-      YandexMystem::Extended.stem('В в в в')
-    end
+
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yandex_mystem
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 3.0.0
 platform: ruby
 authors:
 - Dmitry Polushkin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-28 00:00:00.000000000 Z
+date: 2014-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -43,21 +43,24 @@ description: Mystem is a software that provided by the Yandex only for non-comme
   simple morphological analysis of russian words.
 email:
 - dmitry.polushkin@gmail.com
-executables: []
+executables:
+- mystem-bsd
+- mystem-linux-32
+- mystem-linux-64
+- mystem-win.exe
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
-- ".rvmrc"
 - ".travis.yml"
 - Gemfile
 - README.md
 - Rakefile
-- app/mystem-linux-32
-- app/mystem-linux-64
-- app/mystem-mac
-- app/mystem-win.exe
+- bin/mystem-bsd
+- bin/mystem-linux-32
+- bin/mystem-linux-64
+- bin/mystem-win.exe
 - lib/yandex_mystem.rb
 - lib/yandex_mystem/version.rb
 - spec/spec_helper.rb

data/.rvmrc

@@ -1 +0,0 @@
-rvm use 2.0.0 --create