embulk-filter-kuromoji 0.3.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ae73f70a252f75cff653883c3410115220b0516a
-  data.tar.gz: 239fdb7c9aa444898f660dd280645521c9e1368e
+  metadata.gz: d34a0b4db85a5b6954b4bb957398267e385a4503
+  data.tar.gz: 88ff125a93503f4d0961d85270278f2c666fead1
 SHA512:
-  metadata.gz: 347e0a417a8131a2b504b53ccd4e943bd1e4afd125ee95180efde77c100005b02010a9d9dafa77cfd67b4d0b244b4691b9e64199dddcd0975b14ff42af1f731e
-  data.tar.gz: d5fadea360f8ce2e0d25de2e0a825f60d34eae1b60b88fb50e361eb6e13e8261f7264162c91e94921a73ba18e2c727cf22426a7b1c9b0f38043760e58fc06ca5
+  metadata.gz: ca6c983fc956ba600c5ba89c45ab06826f903c68ef689eabfaeffb29df574d6e7e149a743e197bbfefc8127845397051f39fe4ecebed829ec18ce6633373657a
+  data.tar.gz: b072ee765942a7c72569e435d14322b172cd6acd395e2bc13e99d4103a2a7518039a0c2ee79111475a3880451180f94c47bb9a5103c741b0ed106134197afe39

data/README.md

@@ -1,8 +1,12 @@
 # Kuromoji filter plugin for Embulk
 
 Kuromoji filter plugin for Embulk.
+Neologd support.
 
-see. [Atilika - Applied Search Innovation](http://www.atilika.com/en/products/kuromoji.html)
+## Reference
+
+* [Atilika - Applied Search Innovation](http://www.atilika.com/en/products/kuromoji.html)
+* [Home · neologd/mecab-ipadic-neologd Wiki](https://github.com/neologd/mecab-ipadic-neologd/wiki)
 
 ## Overview
 
@@ -10,6 +14,9 @@ see. [Atilika - Applied Search Innovation](http://www.atilika.com/en/products/ku
 
 ## Configuration
 
+- **tokenizer**: select tokenizer.(kuromoji or neologd) (string, default: kuromoji)
+- **mode**: select mode.(normal or search or extended) (string, default: normal)
+- **use_stop_tag**: neologd only.(bool, default: false)
 - **key_names**: description (list, required)
 - **keep_input**: keep input columns. (bool, default: `true`)
 - **ok_parts_of_speech**: ok parts of speech. (list, default: null)
@@ -20,12 +27,30 @@ see. [Atilika - Applied Search Innovation](http://www.atilika.com/en/products/ku
     - **delimiter**: delimiter (string, default: ",")
     - **type**: extract data type, array or string. array is json type. (string, default: "string")
 
-## Example
+## Neologd Example
+
+```yaml
+filters:
+  - type: kuromoji
+    tokenizer: neologd
+    use_stop_tag: true
+    key_names:
+      - catchcopy
+    settings:
+      - { method: 'reading', delimiter: '' }
+      - { suffix: _surface_form_no_delim, method: 'surface_form', delimiter: '' }
+      - { suffix: _base_form, method: 'base_form', delimiter: '###' }
+      - { suffix: _surface_form, method: 'surface_form', delimiter: '###' }
+      - { suffix: _array, method: 'surface_form', type: 'array' }
+```
+
+## Pure kuromoji Example
 
 ```yaml
 filters:
   - type: kuromoji
     keep_input: false
+    mode: search
     ok_parts_of_speech:
       - 名詞
     key_names:
@@ -75,6 +100,14 @@ As below
       - { suffix: _surface_form, method: 'surface_form', delimiter: '###' }
 ```
 
+## user dictionary exsample
+
+```
+西国分寺,西国分寺,ニシコクブンジ,駅名
+東京スカイツリー,東京 スカイツリー,トウキョウ スカイツリー,カスタム名詞
+```
+
+
 ## Build
 
 ```

data/build.gradle

@@ -9,12 +9,15 @@ import com.github.jrubygradle.JRubyExec
 repositories {
     mavenCentral()
     jcenter()
+    maven {
+        url "http://maven.codelibs.org"
+    }
 }
 configurations {
     provided
 }
 
-version = "0.3.3"
+version = "0.4.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -22,6 +25,7 @@ targetCompatibility = 1.7
 dependencies {
     compile  "org.embulk:embulk-core:0.8.9"
     compile 'com.atilika.kuromoji:kuromoji-ipadic:0.9.0'
+    compile "org.codelibs:lucene-analyzers-kuromoji-ipadic-neologd:5.4.1-20160218"
     provided "org.embulk:embulk-core:0.8.9"
     testCompile "junit:junit:4.+"
     testCompile  "org.embulk:embulk-core:0.8.9"
@@ -76,8 +80,8 @@ Gem::Specification.new do |spec|
   spec.name          = "${project.name}"
   spec.version       = "${project.version}"
   spec.authors       = ["toyama0919"]
-  spec.summary       = %[Kuromoji filter plugin for Embulk]
-  spec.description   = %[Kuromoji filter plugin for Embulk]
+  spec.summary       = %[Kuromoji filter plugin for Embulk. Neologd support.]
+  spec.description   = %[Kuromoji filter plugin for Embulk. Neologd support.]
   spec.email         = ["toyama0919@gmail.com"]
   spec.licenses      = ["MIT"]
   spec.homepage      = "https://github.com/toyama0919/embulk-filter-kuromoji"

data/src/main/java/org/embulk/filter/kuromoji/KuromojiFilterPlugin.java

@@ -31,6 +31,18 @@ public class KuromojiFilterPlugin implements FilterPlugin
         @Config("key_names")
         public List<String> getKeyNames();
 
+        @Config("tokenizer")
+        @ConfigDefault("\"kuromoji\"")
+        public String getTokenizer();
+
+        @Config("mode")
+        @ConfigDefault("\"normal\"")
+        public String getMode();
+
+        @Config("use_stop_tag")
+        @ConfigDefault("false")
+        public boolean getUseStopTag();
+
         @Config("dictionary_path")
         @ConfigDefault("null")
         public Optional<String> getDictionaryPath();
@@ -61,6 +73,11 @@ public class KuromojiFilterPlugin implements FilterPlugin
     @Override
     public PageOutput open(TaskSource taskSource, final Schema inputSchema, final Schema outputSchema, final PageOutput output)
     {
+        final String tokenizer = taskSource.loadTask(PluginTask.class).getTokenizer();
+        logger.info("Tokenizer => {}", tokenizer);
+        if (tokenizer.equals("neologd")){
+            return new NeologdPageOutput(taskSource, inputSchema, outputSchema, output);
+        }
         return new KuromojiPageOutput(taskSource, inputSchema, outputSchema, output);
     }
 

data/src/main/java/org/embulk/filter/kuromoji/KuromojiPageOutput.java

@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.embulk.config.TaskSource;
+import org.embulk.filter.kuromoji.KuromojiFilterPlugin;
 import org.embulk.filter.kuromoji.KuromojiFilterPlugin.PluginTask;
 import org.embulk.spi.Column;
 import org.embulk.spi.Exec;
@@ -19,6 +20,7 @@ import org.msgpack.value.Value;
 import org.msgpack.value.ValueFactory;
 import org.slf4j.Logger;
 
+import com.atilika.kuromoji.TokenizerBase.Mode;
 import com.atilika.kuromoji.ipadic.Token;
 import com.atilika.kuromoji.ipadic.Tokenizer;
 import com.atilika.kuromoji.ipadic.Tokenizer.Builder;
@@ -28,7 +30,7 @@ import com.google.common.collect.Lists;
 
 public class KuromojiPageOutput implements PageOutput
 {
-    private final PluginTask task;
+    private final KuromojiFilterPlugin.PluginTask task;
     private final Tokenizer tokenizer;
     private final List<Column> keyNameColumns;
     private final PageReader reader;
@@ -52,6 +54,17 @@ public class KuromojiPageOutput implements PageOutput
                 e.printStackTrace();
             }
         }
+
+        Mode mode = null;
+        if (task.getMode().equals("normal")) {
+            mode = Mode.NORMAL;
+        } else if (task.getMode().equals("search")) {
+            mode = Mode.SEARCH;
+        } else if (task.getMode().equals("extended")) {
+            mode = Mode.EXTENDED;
+        }
+
+        builder.mode(mode);
         this.tokenizer = builder.build();
         this.keyNameColumns = Lists.newArrayList();
 

data/src/main/java/org/embulk/filter/kuromoji/NeologdPageOutput.java

@@ -0,0 +1,205 @@
+package org.embulk.filter.kuromoji;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.codelibs.neologd.ipadic.lucene.analysis.ja.JapaneseAnalyzer;
+import org.codelibs.neologd.ipadic.lucene.analysis.ja.JapaneseTokenizer;
+import org.codelibs.neologd.ipadic.lucene.analysis.ja.JapaneseTokenizer.Mode;
+import org.codelibs.neologd.ipadic.lucene.analysis.ja.dict.UserDictionary;
+import org.codelibs.neologd.ipadic.lucene.analysis.ja.tokenattributes.BaseFormAttribute;
+import org.codelibs.neologd.ipadic.lucene.analysis.ja.tokenattributes.PartOfSpeechAttribute;
+import org.codelibs.neologd.ipadic.lucene.analysis.ja.tokenattributes.ReadingAttribute;
+import org.embulk.config.TaskSource;
+import org.embulk.filter.kuromoji.KuromojiFilterPlugin.PluginTask;
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.type.Types;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+import org.slf4j.Logger;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Joiner;
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+
+public class NeologdPageOutput implements PageOutput
+{
+    private final KuromojiFilterPlugin.PluginTask task;
+    private final List<Column> keyNameColumns;
+    private final PageReader reader;
+    private final PageBuilder builder;
+    private final Schema inputSchema;
+    private final Schema outputSchema;
+    private final JapaneseAnalyzer japaneseAnalyzer;
+    private static final Logger logger = Exec.getLogger(KuromojiFilterPlugin.class);
+
+    public NeologdPageOutput(TaskSource taskSource, Schema inputSchema, Schema outputSchema, PageOutput output) {
+        this.task = taskSource.loadTask(PluginTask.class);
+        this.inputSchema = inputSchema;
+        this.outputSchema = outputSchema;
+        this.keyNameColumns = Lists.newArrayList();
+
+        for (String keyName : task.getKeyNames()) {
+            this.keyNameColumns.add(outputSchema.lookupColumn(keyName));
+        }
+        this.reader = new PageReader(inputSchema);
+        this.builder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
+
+        UserDictionary userDict = null;
+        if (task.getDictionaryPath().isPresent()) {
+            try {
+                File file = new File(task.getDictionaryPath().get());
+                Reader reader = new InputStreamReader(new FileInputStream(file), Charsets.UTF_8);
+                userDict = UserDictionary.open(reader);
+            } catch (Exception e) {
+                logger.error("neologd error", e);
+            }
+        }
+
+        Mode mode = null;
+        if (task.getMode().equals("normal")) {
+            mode = JapaneseTokenizer.Mode.NORMAL;
+        } else if (task.getMode().equals("search")) {
+            mode = JapaneseTokenizer.Mode.SEARCH;
+        } else if (task.getMode().equals("extended")) {
+            mode = JapaneseTokenizer.Mode.EXTENDED;
+        }
+
+        CharArraySet stopSet = null;
+        Set<String> stopTags = Sets.newHashSet();
+        if (task.getUseStopTag()) {
+            stopSet = JapaneseAnalyzer.getDefaultStopSet();
+            stopTags = JapaneseAnalyzer.getDefaultStopTags();
+        }
+        this.japaneseAnalyzer = new JapaneseAnalyzer(userDict, mode, stopSet, stopTags);
+    }
+
+    @Override
+    public void finish() {
+        builder.finish();
+    }
+
+    @Override
+    public void close() {
+        builder.close();
+    }
+
+    @Override
+    public void add(Page page) {
+        reader.setPage(page);
+        while (reader.nextRecord()) {
+            setValue(builder);
+            builder.addRecord();
+        }
+    }
+
+    /**
+     * @param builder
+     */
+    private void setValue(PageBuilder builder) {
+        if (task.getKeepInput()) {
+            for (Column inputColumn : inputSchema.getColumns()) {
+                if (reader.isNull(inputColumn)) {
+                    builder.setNull(inputColumn);
+                    continue;
+                }
+                if (Types.STRING.equals(inputColumn.getType())) {
+                    builder.setString(inputColumn, reader.getString(inputColumn));
+                } else if (Types.BOOLEAN.equals(inputColumn.getType())) {
+                    builder.setBoolean(inputColumn, reader.getBoolean(inputColumn));
+                } else if (Types.DOUBLE.equals(inputColumn.getType())) {
+                    builder.setDouble(inputColumn, reader.getDouble(inputColumn));
+                } else if (Types.LONG.equals(inputColumn.getType())) {
+                    builder.setLong(inputColumn, reader.getLong(inputColumn));
+                } else if (Types.TIMESTAMP.equals(inputColumn.getType())) {
+                    builder.setTimestamp(inputColumn, reader.getTimestamp(inputColumn));
+                } else if (Types.JSON.equals(inputColumn.getType())) {
+                    builder.setJson(inputColumn, reader.getJson(inputColumn));
+                }
+            }
+        }
+
+        for (Column column : keyNameColumns) {
+            final String source = reader.getString(column);
+            List<Token> tokens = tokenize(new StringReader(source));
+            for (Map<String, String> setting : task.getSettings()) {
+                String suffix = setting.get("suffix");
+                String method = setting.get("method");
+                Column outputColumn = outputSchema.lookupColumn(column.getName() + MoreObjects.firstNonNull(suffix, ""));
+                List<Value> outputs = Lists.newArrayList();
+                for (Token token : tokens) {
+                    String word = null;
+                    if ("base_form".equals(method)) {
+                        word = token.getBaseForm();
+                    } else if ("reading".equals(method)) {
+                        word = token.getReading();
+                    } else if ("surface_form".equals(method)) {
+                        word = token.getCharTerm();
+                    }
+                    if (word != null) {
+                        outputs.add(ValueFactory.newString(word));
+                    }
+                }
+                if (outputColumn.getType().equals(Types.STRING)) {
+                    Joiner joiner = Joiner.on(MoreObjects.firstNonNull(setting.get("delimiter"), ",")).skipNulls();
+                    builder.setString(outputColumn, joiner.join(outputs));
+                } else if (outputColumn.getType().equals(Types.JSON)) {
+                    builder.setJson(outputColumn, ValueFactory.newArray(outputs));
+                }
+            }
+        }
+    }
+
+    private boolean isOkPartsOfSpeech(Token token) {
+        logger.debug("{} => {}", token.getCharTerm(), token.getPartOfSpeech());
+        if (!task.getOkPartsOfSpeech().isPresent()) { return true; };
+        for (String okPartsOfSpeech : task.getOkPartsOfSpeech().get()) {
+            if (token.getPartOfSpeech().startsWith(okPartsOfSpeech)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private List<Token> tokenize(Reader reader) {
+        List<Token> list = Lists.newArrayList();
+        try (TokenStream tokenStream = japaneseAnalyzer.tokenStream("", reader) ) {
+            BaseFormAttribute baseAttr = tokenStream.addAttribute(BaseFormAttribute.class);
+            CharTermAttribute charAttr = tokenStream.addAttribute(CharTermAttribute.class);
+            PartOfSpeechAttribute posAttr = tokenStream.addAttribute(PartOfSpeechAttribute.class);
+            ReadingAttribute readAttr = tokenStream.addAttribute(ReadingAttribute.class);
+
+            tokenStream.reset();
+            while (tokenStream.incrementToken()) {
+                Token token = new Token();
+                token.setCharTerm(charAttr.toString());
+                token.setBaseForm(baseAttr.getBaseForm());
+                token.setReading(readAttr.getReading());
+                token.setPartOfSpeech(posAttr.getPartOfSpeech());
+                if (!isOkPartsOfSpeech(token)) { continue; }
+                list.add(token);
+            }
+        } catch (Exception e) {
+            logger.error("neologd error", e);
+        }
+        return list;
+    }
+}

data/src/main/java/org/embulk/filter/kuromoji/Token.java

@@ -0,0 +1,41 @@
+package org.embulk.filter.kuromoji;
+
+public class Token
+{
+    private String charTerm;
+    private String baseForm;
+    private String partOfSpeech;
+    private String reading;
+    private String inflection;
+
+    public String getCharTerm() {
+        return charTerm;
+    }
+    public String getBaseForm() {
+        return baseForm;
+    }
+    public String getPartOfSpeech() {
+        return partOfSpeech;
+    }
+    public void setCharTerm(String charTerm) {
+        this.charTerm = charTerm;
+    }
+    public void setBaseForm(String baseForm) {
+        this.baseForm = baseForm;
+    }
+    public void setPartOfSpeech(String partOfSpeech) {
+        this.partOfSpeech = partOfSpeech;
+    }
+    public void setReading(String reading) {
+        this.reading = reading;
+    }
+    public String getReading() {
+        return reading;
+    }
+    public String getInflection() {
+        return inflection;
+    }
+    public void setInflection(String inflection) {
+        this.inflection = inflection;
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-kuromoji
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors:
 - toyama0919
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-21 00:00:00.000000000 Z
+date: 2016-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -38,7 +38,7 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-description: Kuromoji filter plugin for Embulk
+description: Kuromoji filter plugin for Embulk. Neologd support.
 email:
 - toyama0919@gmail.com
 executables: []
@@ -58,10 +58,15 @@ files:
 - lib/embulk/filter/kuromoji.rb
 - src/main/java/org/embulk/filter/kuromoji/KuromojiFilterPlugin.java
 - src/main/java/org/embulk/filter/kuromoji/KuromojiPageOutput.java
+- src/main/java/org/embulk/filter/kuromoji/NeologdPageOutput.java
+- src/main/java/org/embulk/filter/kuromoji/Token.java
 - src/test/java/org/embulk/filter/kuromoji/TestKuromojiFilterPlugin.java
-- classpath/embulk-filter-kuromoji-0.3.3.jar
+- classpath/embulk-filter-kuromoji-0.4.0.jar
 - classpath/kuromoji-core-0.9.0.jar
 - classpath/kuromoji-ipadic-0.9.0.jar
+- classpath/lucene-analyzers-common-5.4.1.jar
+- classpath/lucene-analyzers-kuromoji-ipadic-neologd-5.4.1-20160218.jar
+- classpath/lucene-core-5.4.1.jar
 homepage: https://github.com/toyama0919/embulk-filter-kuromoji
 licenses:
 - MIT
@@ -85,6 +90,6 @@ rubyforge_project:
 rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
-summary: Kuromoji filter plugin for Embulk
+summary: Kuromoji filter plugin for Embulk. Neologd support.
 test_files: []
 has_rdoc: