embulk-filter-kuromoji 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4b4dca9e4715f6af3d568ca4013519d4f5351470
-  data.tar.gz: 9acf950dff08f825309b721eb63fefc031977b99
+  metadata.gz: 3d706d852774bb54a883e1fad994b508213b0a12
+  data.tar.gz: c68d3372c01a5b273391f9c14e3b19dce7977aaa
 SHA512:
-  metadata.gz: d5b3f8dda0b8ccc31be015bb839925640fdc109c9fe6de98ced99e9008d2df27dfa8c686148b55d7be0694ad19cf1110db11a3227a5941b69e19c5af9b693d17
-  data.tar.gz: 2eb8b106940f6716b782ef1e5b2960d96d425b475c33d652a797620a40caef87d4157d916cb99d47e1b134766e35212a54a90a201aa8415f3fcb630fe19160e8
+  metadata.gz: 163708524757836fb4a45b5f88b5fb030e7e39d91079d5f2aff2ab7dad31c79d52ba6a199979bea0b7cb73ef32cd6d1abd7e28f7a1c7b4f988a8e0aea6d97b8a
+  data.tar.gz: 8e9a645c381a40a0fa9dfefe6103a4bbfb96347eccf110ba870a488f6dda136be2982bbf6004790d6790839648c15e5ca78b6d5960d084b0a637b4e54e9d7e05

data/build.gradle

@@ -2,27 +2,27 @@ plugins {
     id "com.jfrog.bintray" version "1.1"
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
+    id "checkstyle"
     id "eclipse"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
     mavenCentral()
     jcenter()
-    maven { url "http://www.atilika.org/nexus/content/repositories/atilika" }    
 }
 configurations {
     provided
 }
 
-version = "0.2.0"
+version = "0.3.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.7.4"
-    compile 'org.atilika.kuromoji:kuromoji:0.7.7'
-    provided "org.embulk:embulk-core:0.7.4"
+    compile  "org.embulk:embulk-core:0.8.1"
+    compile 'com.atilika.kuromoji:kuromoji-ipadic:0.9.0'
+    provided "org.embulk:embulk-core:0.8.1"
     testCompile "junit:junit:4.+"
 }
 
@@ -33,6 +33,23 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
 }
 clean { delete "classpath" }
 
+checkstyle {
+    configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
+    toolVersion = '6.14.1'
+}
+checkstyleMain {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+checkstyleTest {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+task checkstyle(type: Checkstyle) {
+    classpath = sourceSets.main.output + sourceSets.test.output
+    source = sourceSets.main.allJava + sourceSets.test.allJava
+}
+
 task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
     jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
     script "${project.name}.gemspec"

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Tue Aug 11 00:26:20 PDT 2015
+#Wed Jan 13 12:41:02 JST 2016
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip

data/src/main/java/org/embulk/filter/kuromoji/KuromojiFilterPlugin.java

@@ -5,9 +5,6 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-import org.atilika.kuromoji.Token;
-import org.atilika.kuromoji.Tokenizer;
-import org.atilika.kuromoji.Tokenizer.Builder;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigSource;
@@ -21,8 +18,14 @@ import org.embulk.spi.PageBuilder;
 import org.embulk.spi.PageOutput;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
+import org.embulk.spi.type.Type;
 import org.embulk.spi.type.Types;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
 
+import com.atilika.kuromoji.ipadic.Token;
+import com.atilika.kuromoji.ipadic.Tokenizer;
+import com.atilika.kuromoji.ipadic.Tokenizer.Builder;
 import com.google.common.base.Joiner;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Optional;
@@ -43,7 +46,7 @@ public class KuromojiFilterPlugin implements FilterPlugin
         @Config("ok_parts_of_speech")
         @ConfigDefault("null")
         public Optional<List<String>> getOkPartsOfSpeech();
-        
+
         @Config("keep_input")
         @ConfigDefault("true")
         public boolean getKeepInput();
@@ -61,21 +64,22 @@ public class KuromojiFilterPlugin implements FilterPlugin
         ImmutableList.Builder<Column> builder = ImmutableList.builder();
         int i = 0;
         if (task.getKeepInput()) {
-            for (Column inputColumn: inputSchema.getColumns()) {
+            for (Column inputColumn : inputSchema.getColumns()) {
                 Column outputColumn = new Column(i++, inputColumn.getName(), inputColumn.getType());
                 builder.add(outputColumn);
             }
         }
 
-        for (String key: task.getKeyNames()) {
+        for (String key : task.getKeyNames()) {
             for (Map<String, String> setting : task.getSettings()) {
                 String keyName = key + MoreObjects.firstNonNull(setting.get("suffix"), "");
+                Type type = "array".equals(setting.get("type")) ? Types.JSON : Types.STRING;
                 if (task.getKeepInput()) {
                     if (setting.get("suffix") != null) {
-                        builder.add(new Column(i++, keyName, Types.STRING));
+                        builder.add(new Column(i++, keyName, type));
                     }
                 } else {
-                    builder.add(new Column(i++, keyName, Types.STRING));
+                    builder.add(new Column(i++, keyName, type));
                 }
             }
         }
@@ -88,7 +92,8 @@ public class KuromojiFilterPlugin implements FilterPlugin
     public PageOutput open(TaskSource taskSource, final Schema inputSchema, final Schema outputSchema, final PageOutput output)
     {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
-        Builder builder = Tokenizer.builder();
+
+        Builder builder = new Tokenizer.Builder();
         if (task.getDictionaryPath().isPresent()) {
             try {
                 builder.userDictionary(task.getDictionaryPath().get());
@@ -102,7 +107,7 @@ public class KuromojiFilterPlugin implements FilterPlugin
         final List<Column> keyNameColumns = Lists.newArrayList();
 
         for (String keyName : task.getKeyNames()) {
-            keyNameColumns.add(inputSchema.lookupColumn(keyName));
+            keyNameColumns.add(outputSchema.lookupColumn(keyName));
         }
 
         return new PageOutput() {
@@ -133,7 +138,7 @@ public class KuromojiFilterPlugin implements FilterPlugin
              */
             private void setValue(PageBuilder builder) {
                 if (task.getKeepInput()) {
-                    for (Column inputColumn: inputSchema.getColumns()) {
+                    for (Column inputColumn : inputSchema.getColumns()) {
                         if (reader.isNull(inputColumn)) {
                             builder.setNull(inputColumn);
                             continue;
@@ -154,29 +159,40 @@ public class KuromojiFilterPlugin implements FilterPlugin
 
                 for (Column column : keyNameColumns) {
                     List<Token> tokens = tokenizer.tokenize(reader.getString(column));
-                    for (Map<String, String> setting: task.getSettings()) {
+                    for (Map<String, String> setting : task.getSettings()) {
                         String suffix = setting.get("suffix");
                         String method = setting.get("method");
                         Column outputColumn = outputSchema.lookupColumn(column.getName() + MoreObjects.firstNonNull(suffix, ""));
-                        List<String> outputs = Lists.newArrayList();
+                        List<Value> outputs = Lists.newArrayList();
                         for (Token token : tokens) {
-                            if (!isOkPartsOfSpeech(token)) continue;
+                            System.err.println(token.getAllFeaturesArray().toString());
+                            System.err.println(token.getPartOfSpeechLevel1());
+                            System.err.println(token.getPartOfSpeechLevel2());
+                            System.err.println(token.getPartOfSpeechLevel3());
+                            System.err.println(token.getPartOfSpeechLevel4());
+                            if (!isOkPartsOfSpeech(token)) { continue; }
+                            String word = null;
                             if ("base_form".equals(method)) {
-                                outputs.add(MoreObjects.firstNonNull(token.getBaseForm(), token.getSurfaceForm()));
+                                word = MoreObjects.firstNonNull(token.getBaseForm(), token.getSurface());
                             } else if ("reading".equals(method)) {
-                                outputs.add(MoreObjects.firstNonNull(token.getReading(), token.getSurfaceForm()));
+                                word = MoreObjects.firstNonNull(token.getReading(), token.getSurface());
                             } else if ("surface_form".equals(method)) {
-                                outputs.add(token.getSurfaceForm());
+                                word = token.getSurface();
                             }
+                            outputs.add(ValueFactory.newString(word));
+                        }
+                        if (outputColumn.getType().equals(Types.STRING)) {
+                            Joiner joiner = Joiner.on(MoreObjects.firstNonNull(setting.get("delimiter"), ",")).skipNulls();
+                            builder.setString(outputColumn, joiner.join(outputs));
+                        } else if (outputColumn.getType().equals(Types.JSON)) {
+                            builder.setJson(outputColumn, ValueFactory.newArray(outputs));
                         }
-                        Joiner joiner = Joiner.on(MoreObjects.firstNonNull(setting.get("delimiter"), ",")).skipNulls();
-                        builder.setString(outputColumn, joiner.join(outputs));
                     }
                 }
             }
 
             private boolean isOkPartsOfSpeech(Token token) {
-                if (!task.getOkPartsOfSpeech().isPresent()) return true;
+                if (!task.getOkPartsOfSpeech().isPresent()) { return true; };
                 for (String okPartsOfSpeech : task.getOkPartsOfSpeech().get()) {
                     if (token.getAllFeaturesArray()[0].equals(okPartsOfSpeech)) {
                         return true;

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-kuromoji
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - toyama0919
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-23 00:00:00.000000000 Z
+date: 2016-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  name: rake
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
 description: Kuromoji filter plugin for Embulk
 email:
 - toyama0919@gmail.com
@@ -56,8 +56,9 @@ files:
 - lib/embulk/filter/kuromoji.rb
 - src/main/java/org/embulk/filter/kuromoji/KuromojiFilterPlugin.java
 - src/test/java/org/embulk/filter/kuromoji/TestKuromojiFilterPlugin.java
-- classpath/embulk-filter-kuromoji-0.2.0.jar
-- classpath/kuromoji-0.7.7.jar
+- classpath/embulk-filter-kuromoji-0.3.0.jar
+- classpath/kuromoji-core-0.9.0.jar
+- classpath/kuromoji-ipadic-0.9.0.jar
 homepage: https://github.com/toyama0919/embulk-filter-kuromoji
 licenses:
 - MIT
@@ -83,4 +84,3 @@ signing_key:
 specification_version: 4
 summary: Kuromoji filter plugin for Embulk
 test_files: []
-has_rdoc: