embulk-filter-kuromoji 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 39f5fef3dee7c66100e57b525ffa310c693cbad6
-  data.tar.gz: 40bc8e9a948364242d091ceb84e9b10b8ec57392
+  metadata.gz: 2fcf8d7ebe4ba3b82de6056dad530ed8254bf358
+  data.tar.gz: 1af3161a76c6f26429f56c09522f3329e1db3745
 SHA512:
-  metadata.gz: af95448ce60356db7f65ad3d27c5fbb6441a46d0dc18df64e47f527ccc2fb92052acbb21ce292e94ab26f6b060f9ac0e4c60976a9764d85681f86b6b70bbf0d6
-  data.tar.gz: 4d0f23b0bee7ea2aff3cdc2785a2bfb4102aa9b390976902f2d0405e6c17ae6dc9cccffbc9efb3043fd97e7f8410f787158f55b6dd887effb8e6f874df5c532b
+  metadata.gz: 38de81bb46e81d51ae79d809d862f5323ac1d6301c7802ef3a51b55a0a19df4c2ce333585672ee71e9eb11a5e9509045d08f527421fe748e670028848b6985f7
+  data.tar.gz: 32e55429d7045a2f62a8dac4cab4a44fa2a373e858c3bdcdb2044f810512ce2c2af1c7ffb314c6c65660658816aed408f4dd8e2d468344b02cc4e51eef918121

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.3.1"
+version = "0.3.2"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/filter/kuromoji/KuromojiFilterPlugin.java

@@ -1,7 +1,5 @@
 package org.embulk.filter.kuromoji;
 
-import java.io.FileNotFoundException;
-import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
@@ -13,28 +11,21 @@ import org.embulk.config.TaskSource;
 import org.embulk.spi.Column;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FilterPlugin;
-import org.embulk.spi.Page;
-import org.embulk.spi.PageBuilder;
 import org.embulk.spi.PageOutput;
-import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.type.Type;
 import org.embulk.spi.type.Types;
-import org.msgpack.value.Value;
-import org.msgpack.value.ValueFactory;
+import org.slf4j.Logger;
 
-import com.atilika.kuromoji.ipadic.Token;
-import com.atilika.kuromoji.ipadic.Tokenizer;
-import com.atilika.kuromoji.ipadic.Tokenizer.Builder;
-import com.google.common.base.Joiner;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
 public class KuromojiFilterPlugin implements FilterPlugin
 {
+    private static final Logger logger = Exec.getLogger(KuromojiFilterPlugin.class);
+
     public interface PluginTask extends Task
     {
         @Config("key_names")
@@ -62,6 +53,34 @@ public class KuromojiFilterPlugin implements FilterPlugin
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
+        Schema outputSchema = buildOutputSchema(task, inputSchema);
+
+        control.run(task.dump(), outputSchema);
+    }
+
+    @Override
+    public PageOutput open(TaskSource taskSource, final Schema inputSchema, final Schema outputSchema, final PageOutput output)
+    {
+        return new KuromojiPageOutput(taskSource, inputSchema, outputSchema, output);
+    }
+
+    /**
+     * @param inputSchema
+     * @param task
+     * @return
+     */
+    private Schema buildOutputSchema(PluginTask task, Schema inputSchema) {
+        final List<Column> outputColumns = buildOutputColumns(task, inputSchema);
+        logger.debug("outputColumns => {}", outputColumns);
+        return new Schema(outputColumns);
+    }
+
+    /**
+     * @param task
+     * @param inputSchema
+     * @return
+     */
+    private List<Column> buildOutputColumns(PluginTask task, Schema inputSchema) {
         ImmutableList.Builder<Column> builder = ImmutableList.builder();
         Map<String, Column> map = Maps.newHashMap();
         int i = 0;
@@ -85,120 +104,6 @@ public class KuromojiFilterPlugin implements FilterPlugin
             final Column column = e.getValue();
             builder.add(new Column(i++, column.getName(), column.getType()));
         }
-
-        Schema outputSchema = new Schema(builder.build());
-        control.run(task.dump(), outputSchema);
-    }
-
-    @Override
-    public PageOutput open(TaskSource taskSource, final Schema inputSchema, final Schema outputSchema, final PageOutput output)
-    {
-        final PluginTask task = taskSource.loadTask(PluginTask.class);
-
-        Builder builder = new Tokenizer.Builder();
-        if (task.getDictionaryPath().isPresent()) {
-            try {
-                builder.userDictionary(task.getDictionaryPath().get());
-            } catch (FileNotFoundException e) {
-                e.printStackTrace();
-            } catch (IOException e) {
-                e.printStackTrace();
-            }
-        }
-        final Tokenizer tokenizer = builder.build();
-        final List<Column> keyNameColumns = Lists.newArrayList();
-
-        for (String keyName : task.getKeyNames()) {
-            keyNameColumns.add(outputSchema.lookupColumn(keyName));
-        }
-
-        return new PageOutput() {
-            private PageReader reader = new PageReader(inputSchema);
-            private PageBuilder builder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
-
-            @Override
-            public void finish() {
-                builder.finish();
-            }
-
-            @Override
-            public void close() {
-                builder.close();
-            }
-
-            @Override
-            public void add(Page page) {
-                reader.setPage(page);
-                while (reader.nextRecord()) {
-                    setValue(builder);
-                    builder.addRecord();
-                }
-            }
-
-            /**
-             * @param builder
-             */
-            private void setValue(PageBuilder builder) {
-                if (task.getKeepInput()) {
-                    for (Column inputColumn : inputSchema.getColumns()) {
-                        if (reader.isNull(inputColumn)) {
-                            builder.setNull(inputColumn);
-                            continue;
-                        }
-                        if (Types.STRING.equals(inputColumn.getType())) {
-                            builder.setString(inputColumn, reader.getString(inputColumn));
-                        } else if (Types.BOOLEAN.equals(inputColumn.getType())) {
-                            builder.setBoolean(inputColumn, reader.getBoolean(inputColumn));
-                        } else if (Types.DOUBLE.equals(inputColumn.getType())) {
-                            builder.setDouble(inputColumn, reader.getDouble(inputColumn));
-                        } else if (Types.LONG.equals(inputColumn.getType())) {
-                            builder.setLong(inputColumn, reader.getLong(inputColumn));
-                        } else if (Types.TIMESTAMP.equals(inputColumn.getType())) {
-                            builder.setTimestamp(inputColumn, reader.getTimestamp(inputColumn));
-                        } else if (Types.JSON.equals(inputColumn.getType())) {
-                            builder.setJson(inputColumn, reader.getJson(inputColumn));
-                        }
-                    }
-                }
-
-                for (Column column : keyNameColumns) {
-                    List<Token> tokens = tokenizer.tokenize(reader.getString(column));
-                    for (Map<String, String> setting : task.getSettings()) {
-                        String suffix = setting.get("suffix");
-                        String method = setting.get("method");
-                        Column outputColumn = outputSchema.lookupColumn(column.getName() + MoreObjects.firstNonNull(suffix, ""));
-                        List<Value> outputs = Lists.newArrayList();
-                        for (Token token : tokens) {
-                            if (!isOkPartsOfSpeech(token)) { continue; }
-                            String word = null;
-                            if ("base_form".equals(method)) {
-                                word = MoreObjects.firstNonNull(token.getBaseForm(), token.getSurface());
-                            } else if ("reading".equals(method)) {
-                                word = MoreObjects.firstNonNull(token.getReading(), token.getSurface());
-                            } else if ("surface_form".equals(method)) {
-                                word = token.getSurface();
-                            }
-                            outputs.add(ValueFactory.newString(word));
-                        }
-                        if (outputColumn.getType().equals(Types.STRING)) {
-                            Joiner joiner = Joiner.on(MoreObjects.firstNonNull(setting.get("delimiter"), ",")).skipNulls();
-                            builder.setString(outputColumn, joiner.join(outputs));
-                        } else if (outputColumn.getType().equals(Types.JSON)) {
-                            builder.setJson(outputColumn, ValueFactory.newArray(outputs));
-                        }
-                    }
-                }
-            }
-
-            private boolean isOkPartsOfSpeech(Token token) {
-                if (!task.getOkPartsOfSpeech().isPresent()) { return true; };
-                for (String okPartsOfSpeech : task.getOkPartsOfSpeech().get()) {
-                    if (token.getAllFeaturesArray()[0].equals(okPartsOfSpeech)) {
-                        return true;
-                    }
-                }
-                return false;
-            }
-        };
+        return builder.build();
     }
 }

data/src/main/java/org/embulk/filter/kuromoji/KuromojiPageOutput.java

@@ -0,0 +1,151 @@
+package org.embulk.filter.kuromoji;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.embulk.config.TaskSource;
+import org.embulk.filter.kuromoji.KuromojiFilterPlugin.PluginTask;
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.type.Types;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+import org.slf4j.Logger;
+
+import com.atilika.kuromoji.ipadic.Token;
+import com.atilika.kuromoji.ipadic.Tokenizer;
+import com.atilika.kuromoji.ipadic.Tokenizer.Builder;
+import com.google.common.base.Joiner;
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.Lists;
+
+public class KuromojiPageOutput implements PageOutput
+{
+    private final PluginTask task;
+    private final Tokenizer tokenizer;
+    private final List<Column> keyNameColumns;
+    private final PageReader reader;
+    private final PageBuilder builder;
+    private final Schema inputSchema;
+    private final Schema outputSchema;
+    private static final Logger logger = Exec.getLogger(KuromojiFilterPlugin.class);
+
+    public KuromojiPageOutput(TaskSource taskSource, Schema inputSchema, Schema outputSchema, PageOutput output) {
+        this.task = taskSource.loadTask(PluginTask.class);
+        this.inputSchema = inputSchema;
+        this.outputSchema = outputSchema;
+
+        Builder builder = new Tokenizer.Builder();
+        if (task.getDictionaryPath().isPresent()) {
+            try {
+                builder.userDictionary(task.getDictionaryPath().get());
+            } catch (FileNotFoundException e) {
+                e.printStackTrace();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+        this.tokenizer = builder.build();
+        this.keyNameColumns = Lists.newArrayList();
+
+        for (String keyName : task.getKeyNames()) {
+            this.keyNameColumns.add(outputSchema.lookupColumn(keyName));
+        }
+        this.reader = new PageReader(inputSchema);
+        this.builder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
+    }
+
+    @Override
+    public void finish() {
+        builder.finish();
+    }
+
+    @Override
+    public void close() {
+        builder.close();
+    }
+
+    @Override
+    public void add(Page page) {
+        reader.setPage(page);
+        while (reader.nextRecord()) {
+            setValue(builder);
+            builder.addRecord();
+        }
+    }
+
+    /**
+     * @param builder
+     */
+    private void setValue(PageBuilder builder) {
+        if (task.getKeepInput()) {
+            for (Column inputColumn : inputSchema.getColumns()) {
+                if (reader.isNull(inputColumn)) {
+                    builder.setNull(inputColumn);
+                    continue;
+                }
+                if (Types.STRING.equals(inputColumn.getType())) {
+                    builder.setString(inputColumn, reader.getString(inputColumn));
+                } else if (Types.BOOLEAN.equals(inputColumn.getType())) {
+                    builder.setBoolean(inputColumn, reader.getBoolean(inputColumn));
+                } else if (Types.DOUBLE.equals(inputColumn.getType())) {
+                    builder.setDouble(inputColumn, reader.getDouble(inputColumn));
+                } else if (Types.LONG.equals(inputColumn.getType())) {
+                    builder.setLong(inputColumn, reader.getLong(inputColumn));
+                } else if (Types.TIMESTAMP.equals(inputColumn.getType())) {
+                    builder.setTimestamp(inputColumn, reader.getTimestamp(inputColumn));
+                } else if (Types.JSON.equals(inputColumn.getType())) {
+                    builder.setJson(inputColumn, reader.getJson(inputColumn));
+                }
+            }
+        }
+
+        for (Column column : keyNameColumns) {
+            final String source = reader.getString(column);
+            List<Token> tokens = tokenizer.tokenize(source);
+            logger.debug("{} => {}", source, tokens);
+            for (Map<String, String> setting : task.getSettings()) {
+                String suffix = setting.get("suffix");
+                String method = setting.get("method");
+                Column outputColumn = outputSchema.lookupColumn(column.getName() + MoreObjects.firstNonNull(suffix, ""));
+                List<Value> outputs = Lists.newArrayList();
+                for (Token token : tokens) {
+                    logger.debug("token => {}, {}", token, token.getAllFeatures());
+                    if (!isOkPartsOfSpeech(token)) { continue; }
+                    String word = null;
+                    if ("base_form".equals(method)) {
+                        word = MoreObjects.firstNonNull(token.getBaseForm(), token.getSurface());
+                    } else if ("reading".equals(method)) {
+                        word = MoreObjects.firstNonNull(token.getReading(), token.getSurface());
+                    } else if ("surface_form".equals(method)) {
+                        word = token.getSurface();
+                    }
+                    outputs.add(ValueFactory.newString(word));
+                }
+                if (outputColumn.getType().equals(Types.STRING)) {
+                    Joiner joiner = Joiner.on(MoreObjects.firstNonNull(setting.get("delimiter"), ",")).skipNulls();
+                    builder.setString(outputColumn, joiner.join(outputs));
+                } else if (outputColumn.getType().equals(Types.JSON)) {
+                    builder.setJson(outputColumn, ValueFactory.newArray(outputs));
+                }
+            }
+        }
+    }
+
+    private boolean isOkPartsOfSpeech(Token token) {
+        if (!task.getOkPartsOfSpeech().isPresent()) { return true; };
+        for (String okPartsOfSpeech : task.getOkPartsOfSpeech().get()) {
+            if (token.getAllFeaturesArray()[0].equals(okPartsOfSpeech)) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-kuromoji
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - toyama0919
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-13 00:00:00.000000000 Z
+date: 2016-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -57,8 +57,9 @@ files:
 - gradlew.bat
 - lib/embulk/filter/kuromoji.rb
 - src/main/java/org/embulk/filter/kuromoji/KuromojiFilterPlugin.java
+- src/main/java/org/embulk/filter/kuromoji/KuromojiPageOutput.java
 - src/test/java/org/embulk/filter/kuromoji/TestKuromojiFilterPlugin.java
-- classpath/embulk-filter-kuromoji-0.3.1.jar
+- classpath/embulk-filter-kuromoji-0.3.2.jar
 - classpath/kuromoji-core-0.9.0.jar
 - classpath/kuromoji-ipadic-0.9.0.jar
 homepage: https://github.com/toyama0919/embulk-filter-kuromoji