twkorean 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9fa5c2b3f783010b5b67c23db599ffd3ec75ef61
-  data.tar.gz: 2117abd053395906001c822082aef413c91441c2
+  metadata.gz: 36d821bc142a63e34c0e143cb53ed2e72f0f72f9
+  data.tar.gz: 274df73ac337c3f01090f942c4b5f3948498470c
 SHA512:
-  metadata.gz: 7ed15b2886e42e367c0652e1a86aad07b98c09279b37e0cd02b1e733499076e69c9481573e91c51d01ae70ed4e3feec4d0610e8dd0bed059d5a536c9193ce776
-  data.tar.gz: 854b177257cf8252e1a81916a02ea721e01ef94ea3e3d4a4d02b02e65a9872bb30934d46c0b7c24a3cb8a8ae5fbca06dceb688496807a4d0f24a6c65ece08198
+  metadata.gz: b3eb5720642faf5de2e04cc2c2d035f15da4889ed6f723facdd2771f9f4f404d52d09dffc1cc8f39a911d4456c62041c51a697c45c7410b8a4e8ae69c151b46f
+  data.tar.gz: 9c80732be97e998f09ff640fc7bbedbb6ddc6f412805386cb1b7b3c8988597331814a6af66ab4e6247b404f7f2525dae9264c11eb972afff3e2517b8d8a6dc1a

data/README.md

@@ -2,7 +2,7 @@
 
 ## Compatibility
 
-Currently wraps [twitter-korean-text 3.0](https://github.com/twitter/twitter-korean-text/tree/korean-text-3.0) / 현재 이 프로젝트는 [twitter-korean-text 3.0](https://github.com/twitter/twitter-korean-text/tree/korean-text-3.0)을 사용중입니다.
+Currently wraps [twitter-korean-text 4.4](https://github.com/twitter/twitter-korean-text/tree/korean-text-4.4) / 현재 이 프로젝트는 [twitter-korean-text 4.4](https://github.com/twitter/twitter-korean-text/tree/korean-text-4.4)을 사용중입니다.
 
 ## Installation
 
@@ -29,41 +29,37 @@ Or install it yourself as:
 
 ## Usage
 
-     describe "Twkorean" do
-        TEXT = "한국어를 처리하는 예시입니닼ㅋㅋㅋㅋㅋ #한국어"
-        before do
-          
-        end
-
-        it "Normalize" do
-          twkorean = Twkorean::TwitterKoreanText.new
-          p "Normlize"
-          p twkorean.normalize(TEXT)
-          # 한국어를 처리하는 예시입니다ㅋㅋ #한국어
-        end
-
-        it "Tokenize" do
-          twkorean = Twkorean::TwitterKoreanText.new(true, false)
-          p "#Tokenize"
-          p twkorean.tokenize(TEXT)
-          # ["한국어(Noun: 0, 3)", "를(Josa: 3, 1)", "처리(Noun: 5, 2)", "하는(Verb: 7, 2)", "예시(Noun: 10, 2)", "입니(Adjective: 12, 2)", "다(Eomi: 14, 1)", "ㅋㅋ(KoreanParticle: 15, 2)", "#한국어(Hashtag: 18, 4)"]
-        end
-
-        it "Stemming" do
-          twkorean = Twkorean::TwitterKoreanText.new
-          p "#Stemming"
-          p twkorean.tokenize(TEXT)
-          # ["한국어(Noun: 0, 3)", "를(Josa: 3, 1)", "처리(Noun: 5, 2)", "하다(Verb: 7, 2)", "예시(Noun: 10, 2)", "이다(Adjective: 12, 3)", "ㅋㅋ(KoreanParticle: 15, 2)", "#한국어(Hashtag: 18, 4)"]
-        end
-
-        it "Phrase extraction" do
-          twkorean = Twkorean::TwitterKoreanText.new
-          p "Phrase extraction"
-          p twkorean.extract_phrases(TEXT)
-          # ["한국어(Noun: 0, 3)", "처리(Noun: 5, 2)", "처리하는 예시(Noun: 5, 7)", "예시(Noun: 10, 2)", "#한국어(Hashtag: 18, 4)"]
-        end
-     end
-## Contributing
+    describe "Twkorean" do
+      text = "한국어를 처리하는 예시입니닼ㅋㅋㅋㅋㅋ #한국어"
+      twkorean = Twkorean::TwitterKoreanText.new
+      text = twkorean.normalize(text)
+      tokens = twkorean.tokenize(text)
+    
+      it "Tokenize" do
+        p "#Tokenize"
+        p twkorean.tokens_to_string_list(tokens)
+        # ["한국어", "를", "처리", "하는", "예시", "입니", "다", "ㅋㅋ", "#한국어"]
+        p twkorean.tokens_to_token_list(tokens)
+        # ["한국어(Noun: 0, 3)", "를(Josa: 3, 1)", "처리(Noun: 5, 2)", "하는(Verb: 7, 2)", "예시(Noun: 10, 2)", "입니(Adjective: 12, 2)", "다(Eomi: 14, 1)", "ㅋㅋ(KoreanParticle: 15, 2)", "#한국어(Hashtag: 18, 4)"]
+      end
+    
+      it "Stemming" do
+        p "#Stemming"
+        stem = twkorean.stem(tokens)
+        p twkorean.tokens_to_string_list(stem)
+        # ["한국어", "를", "처리", "하다", "예시", "이다", "ㅋㅋ", "#한국어"]
+        p twkorean.tokens_to_token_list(stem)
+        # ["한국어(Noun: 0, 3)", "를(Josa: 3, 1)", "처리(Noun: 5, 2)", "하다(Verb: 7, 2)", "예시(Noun: 10, 2)", "이다(Adjective: 12, 3)", "ㅋㅋ(KoreanParticle: 15, 2)", "#한국어(Hashtag: 18, 4)"]
+      end
+    
+      it "Phrase extraction" do
+        p "Phrase extraction"
+        p twkorean.extract_phrases(tokens)
+        # ["한국어(Noun: 0, 3)", "처리(Noun: 5, 2)", "처리하는 예시(Noun: 5, 7)", "예시(Noun: 10, 2)", "#한국어(Hashtag: 18, 4)"]
+      end
+    end
+
+end## Contributing
 
 1. Fork it ( https://github.com/[my-github-username]/twkorean/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)

data/lib/twkorean.rb

@@ -1,6 +1,6 @@
 # @name                twkorean-ruby
 # @author              JunSangPil
-# @version             0.0.3
+# @version             0.0.4
 # @url                 https://github.com/jun85664396/twkorean-ruby
 # @license             Apache License 2.0
 require "twkorean/version"

data/lib/twkorean/twitter_korean_text.rb

@@ -1,6 +1,6 @@
 # @name                twkorean-ruby
 # @author              JunSangPil
-# @version             0.0.3
+# @version             0.0.4
 # @url                 https://github.com/jun85664396/twkorean-ruby
 # @license             Apache License 2.0
 module Twkorean
@@ -11,14 +11,7 @@ module Twkorean
     def initialize(normalization = true, stemming = true)
       jars = Dir.glob(File.dirname(__FILE__)+"/../jars/*.jar").join(':')
       Rjb::load(jars, ['-Xmx512M'])
-      korean_processor = Rjb::import('com.twitter.penguin.korean.TwitterKoreanProcessorJava$Builder').new
-      unless normalization
-        korean_processor.disableNormalizer
-      end
-      unless stemming
-        korean_processor.disableStemmer
-      end
-        self.korean_processor = korean_processor.build
+      self.korean_processor = Rjb::import('com.twitter.penguin.korean.TwitterKoreanProcessorJava')
     end
 
     def normalize(text)
@@ -27,19 +20,26 @@ module Twkorean
 
     def tokenize(text)
       tokens = self.korean_processor.tokenize(text)
-      return [] unless tokens
-      tokens.toArray.map{|x| x.toString}
+      tokens
     end
 
-    def tokenize_to_strings(text)
-      tokens = self.korean_processor.tokenizeToStrings(text)
-      return [] unless tokens
+    def tokens_to_string_list(tokens)
+      tokens = self.korean_processor.tokensToJavaStringList(tokens)
       tokens.toArray.map{|x| x.toString}
     end
 
-    def extract_phrases(text)
-      phrases = self.korean_processor.extractPhrases(text)
-      return [] unless phrases
+    def tokens_to_token_list(tokens)
+      tokens = self.korean_processor.tokensToJavaKoreanTokenList(tokens)
+      tokens.toArray.map{|x| self.parser(x.toString)}
+    end
+
+    def stem(tokens)
+      stemmed = self.korean_processor.stem(tokens)
+      stemmed
+    end
+
+    def extract_phrases(tokens)
+      phrases = self.korean_processor.extractPhrases(tokens, true, true)
       phrases.toArray.map{|x| x.toString}
     end
 

data/lib/twkorean/version.rb

@@ -1,8 +1,8 @@
 # @name                twkorean-ruby
 # @author              JunSangPil
-# @version             0.0.3
+# @version             0.0.4
 # @url                 https://github.com/jun85664396/twkorean-ruby
 # @license             Apache License 2.0
 module Twkorean
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/test/test_helper.rb

@@ -1,6 +1,6 @@
 # @name                twkorean-ruby
 # @author              JunSangPil
-# @version             0.0.3
+# @version             0.0.4
 # @url                 https://github.com/jun85664396/twkorean-ruby
 # @license             Apache License 2.0
 require 'minitest/autorun'

data/test/twkorean.rb

@@ -1,50 +1,38 @@
 # @name                twkorean-ruby
 # @author              JunSangPil
-# @version             0.0.3
+# @version             0.0.4
 # @url                 https://github.com/jun85664396/twkorean-ruby
 # @license             Apache License 2.0
 require_relative 'test_helper'
 require 'twkorean'
 
 describe "Twkorean" do
-  TEXT = "한국어를 처리하는 예시입니닼ㅋㅋㅋㅋㅋ #한국어"
-  before do
-
-  end
-
-  it "Normalize" do
-    twkorean = Twkorean::TwitterKoreanText.new
-    p "Normlize"
-    p twkorean.normalize(TEXT)
-    # 한국어를 처리하는 예시입니다ㅋㅋ #한국어
-  end
+  text = "한국어를 처리하는 예시입니닼ㅋㅋㅋㅋㅋ #한국어"
+  twkorean = Twkorean::TwitterKoreanText.new
+  text = twkorean.normalize(text)
+  tokens = twkorean.tokenize(text)
 
   it "Tokenize" do
-    twkorean = Twkorean::TwitterKoreanText.new(true, false)
     p "#Tokenize"
-    p twkorean.tokenize(TEXT)
+    p twkorean.tokens_to_string_list(tokens)
+    # ["한국어", "를", "처리", "하는", "예시", "입니", "다", "ㅋㅋ", "#한국어"]
+    p twkorean.tokens_to_token_list(tokens)
     # ["한국어(Noun: 0, 3)", "를(Josa: 3, 1)", "처리(Noun: 5, 2)", "하는(Verb: 7, 2)", "예시(Noun: 10, 2)", "입니(Adjective: 12, 2)", "다(Eomi: 14, 1)", "ㅋㅋ(KoreanParticle: 15, 2)", "#한국어(Hashtag: 18, 4)"]
   end
 
   it "Stemming" do
-    twkorean = Twkorean::TwitterKoreanText.new
     p "#Stemming"
-    p twkorean.tokenize(TEXT)
+    stem = twkorean.stem(tokens)
+    p twkorean.tokens_to_string_list(stem)
+    # ["한국어", "를", "처리", "하다", "예시", "이다", "ㅋㅋ", "#한국어"]
+    p twkorean.tokens_to_token_list(stem)
     # ["한국어(Noun: 0, 3)", "를(Josa: 3, 1)", "처리(Noun: 5, 2)", "하다(Verb: 7, 2)", "예시(Noun: 10, 2)", "이다(Adjective: 12, 3)", "ㅋㅋ(KoreanParticle: 15, 2)", "#한국어(Hashtag: 18, 4)"]
   end
 
   it "Phrase extraction" do
-    twkorean = Twkorean::TwitterKoreanText.new
     p "Phrase extraction"
-    p twkorean.extract_phrases(TEXT)
+    p twkorean.extract_phrases(tokens)
     # ["한국어(Noun: 0, 3)", "처리(Noun: 5, 2)", "처리하는 예시(Noun: 5, 7)", "예시(Noun: 10, 2)", "#한국어(Hashtag: 18, 4)"]
   end
 
-  it "Parser" do
-    twkorean = Twkorean::TwitterKoreanText.new(true, false)
-    p "#Tokenize Parser"
-    p twkorean.tokenize(TEXT).map{|x| twkorean.parser(x) }
-    # ["한국어(Noun: 0, 3)", "를(Josa: 3, 1)", "처리(Noun: 5, 2)", "하는(Verb: 7, 2)", "예시(Noun: 10, 2)", "입니(Adjective: 12, 2)", "다(Eomi: 14, 1)", "ㅋㅋ(KoreanParticle: 15, 2)", "#한국어(Hashtag: 18, 4)"]
-  end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twkorean
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - JunSangPil
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-05 00:00:00.000000000 Z
+date: 2016-02-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -78,9 +78,9 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- lib/jars/korean-text-3.0.jar
+- lib/jars/korean-text-4.4.jar
 - lib/jars/scala-library-2.11.6.jar
-- lib/jars/twitter-text-1.11.1.jar
+- lib/jars/twitter-text-1.13.3.jar
 - lib/twkorean.rb
 - lib/twkorean/twitter_korean_text.rb
 - lib/twkorean/version.rb