tiny-classifier 1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ef03bed349d3c266bab512446b6dff8946d5c14f
+  data.tar.gz: 768126f03759f1fe38ef2e155dad3484836f4422
+SHA512:
+  metadata.gz: 9c59f06bef1b3fcc30914a410c29ac52444ea77790f5f4f588515c268bb181a2f353083bfe1ab3ceeba94964052807a6cb556ded863712b09f6488c7d1feff07
+  data.tar.gz: aa188deccf010883b6d04f3ae63ba0124d9ff4f586690cfccf921be5e1466deed11e72b021083142d0e37134f80a1b462af4e7989af0d3c69c830a19391000c9

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,58 @@
+# README
+
+## Name
+
+tiny-classifier
+
+## Description
+
+Command line tool to run text classifier based on naive bayes.
+
+## Install
+
+```
+% gem install tiny-classifier
+```
+
+## Basic usage
+
+Training:
+
+```
+% echo "Hello, world!"        | tc-train --labels=positive,negative positive
+% echo "I'm very very happy!" | tc-train --labels=positive,negative positive
+% echo "I'm so bad..."        | tc-train --labels=positive,negative negative
+% echo "Oh my god!"           | tc-train --labels=positive,negative negative
+```
+
+Classifying:
+
+~~~
+% echo "Happy day?" | tc-classify --labels=positive,negative
+positive
+~~~
+
+## Command line parameters
+
+### Common
+
+`--labels=LABELS` (required)
+:  A comman-separated list of labels. You should use only alphabetic characters. (Non-alphabetical characters will cause problems.)
+
+`--data-dir=PATH` (optional)
+: The path to the directory that the training data to be saved. The current directory.
+
+`--tokenizer=TOKENIZER` (optional)
+: Tokenizer for input which is not separated by whitespaces. Possible values are: only `mecab`.
+
+### Trainer
+
+The `tc-train` requires one command line argument: the label. You need to specify one of labels given via the `--labels` parameter.
+
+## Copyright
+
+Copyright (c) 2017 YUKI "Piro" Hiroshi
+
+## License
+
+GPLv3 or later. See LICENSE.txt for details.

data/Rakefile

@@ -0,0 +1,28 @@
+# Copyright (C) 2017 YUKI "Piro" Hiroshi
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+task :default => :build
+
+require "bundler/gem_helper"
+
+base_dir = File.join(File.dirname(__FILE__))
+
+helper = Bundler::GemHelper.new(base_dir)
+def helper.version_tag
+  version
+end
+
+helper.install
+spec = helper.gemspec

data/bin/tc-classify

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+# Copyright (C) 2017 YUKI "Piro" Hiroshi
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require "tiny-classifier/classifier"
+
+Classifier.run

data/bin/tc-train

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+# Copyright (C) 2017 YUKI "Piro" Hiroshi
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require "tiny-classifier/trainer"
+
+Trainer.run

data/lib/tiny-classifier/base.rb

@@ -0,0 +1,129 @@
+# Copyright (C) 2017 YUKI "Piro" Hiroshi
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require "pathname"
+require "optparse"
+require "classifier-reborn"
+
+class TinyClassifierBase
+  TOKENIZERS = [:none, :mecab]
+
+  def initialize
+    @tokenizer = :none
+    @data_dir = Dir.pwd
+  end
+
+  def parse_command_line_options(command_line_options)
+    option_parser = create_option_parser
+    option_parser.parse!(command_line_options)
+  end
+
+  private
+  def create_option_parser
+    parser = OptionParser.new
+
+    parser.on("--data-dir=PATH",
+              "Path to the directory to store training data file (default=current directory)") do |data_dir|
+      @data_dir = data_dir
+    end
+
+    parser.on("--labels=LABELS",
+              "List of labels (comma-separated)") do |labels|
+      @labels = normalize_labels(labels)
+    end
+
+    parser.on("--tokenizer=TOKENIZER",
+              "Tokenizer (default=#{@tokenizer})") do |tokenizer|
+      @tokenizer = tokenizer.to_sym
+    end
+
+    parser
+  end
+
+  def normalize_labels(labels)
+    labels
+      .strip
+      .downcase
+      .split(",")
+      .collect(&:strip)
+      .reject do |label|
+        label.empty?
+      end
+      .sort
+      .collect(&:capitalize)
+  end
+
+  def data_file_name
+    @data_file_basename ||= prepare_data_file_name
+  end
+
+  def prepare_data_file_name
+    labels = @labels.join("-").downcase
+    "tc.#{labels}.dat"
+  end
+
+  def data_file_path
+    @data_file_path ||= prepare_data_file_path
+  end
+
+  def prepare_data_file_path
+    path = Pathname(@data_dir)
+    path + data_file_name
+  end
+
+  def classifier
+    @classifier ||= prepare_classifier
+  end
+
+  def prepare_classifier
+    if data_file_path.exist?
+      data = File.read(data_file_path.to_s)
+      Marshal.load(data)
+    else
+      ClassifierReborn::Bayes.new(*@labels)
+    end
+  end
+
+  def prepare_input
+    tokenize
+    @input.strip!
+  end
+
+  def tokenize
+    case @tokenizer
+    when :mecab
+      tokenize_by_mecab
+    end
+  end
+
+  def tokenize_by_mecab
+    require "natto"
+    natto = Natto::MeCab.new
+    terms = []
+    natto.parse(@input) do |term|
+      if term.feature =~ /名詞|形容詞|動詞/
+        terms << term.surface
+      end
+    end
+    @input = terms.join(" ").strip
+  end
+
+  def save
+    data = Marshal.dump(classifier)
+    File.open(data_file_path, "w") do |file|
+      file.write(data)
+    end
+  end
+end

data/lib/tiny-classifier/classifier.rb

@@ -0,0 +1,41 @@
+# Copyright (C) 2017 YUKI "Piro" Hiroshi
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require "tiny-classifier/base"
+
+class Classifier < TinyClassifierBase
+  class << self
+    def run(argv=nil)
+      argv ||= ARGV.dup
+      classifier = new
+      classifier.parse_command_line_options(argv)
+      input = $stdin.readlines.join("\n")
+      classifier.run(input: input)
+    end
+  end
+
+  def run(params)
+    @label = params[:label]
+    @input = params[:input]
+    prepare_input
+    if @input.empty?
+      exit(1)
+    else
+      label = classifier.classify(@input)
+      puts label.downcase
+      exit(0)
+    end
+  end
+end

data/lib/tiny-classifier/trainer.rb

@@ -0,0 +1,50 @@
+# Copyright (C) 2017 YUKI "Piro" Hiroshi
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require "tiny-classifier/base"
+
+class Trainer < TinyClassifierBase
+  class << self
+    def run(argv=nil)
+      argv ||= ARGV.dup
+      trainer = new
+      *labels = trainer.parse_command_line_options(argv)
+      input = $stdin.readlines.join("\n")
+      trainer.run(label: labels.first,
+                  input: input)
+    end
+  end
+
+  def run(params)
+    @label = params[:label]
+    @input = params[:input]
+    prepare_input
+    if @input.empty?
+      exit(1)
+    else
+      classifier.send("train_#{@label.downcase}", @input)
+      save
+      exit(0)
+    end
+  end
+
+  private
+  def save
+    data = Marshal.dump(classifier)
+    File.open(data_file_path, "w") do |file|
+      file.write(data)
+    end
+  end
+end

data/tiny-classifier.gemspec

@@ -0,0 +1,42 @@
+# Copyright (C) 2017 YUKI "Piro" Hiroshi
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+clean_white_space = lambda do |entry|
+  entry.gsub(/(\A\n+|\n+\z)/, '') + "\n"
+end
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "lib"))
+
+Gem::Specification.new do |spec|
+  spec.name = "tiny-classifier"
+  spec.version = "1.0"
+  spec.homepage = "https://github.com/piroor/tiny-classifier"
+  spec.authors = ["YUKI \"Piro\" Hiroshi"]
+  spec.email = ["piro.outsider.reflex@gmail.com"]
+  readme = File.read("README.md")
+  readme.force_encoding("UTF-8")
+  entries = readme.split(/^\#\#\s(.*)$/)
+  description = clean_white_space.call(entries[entries.index("Description") + 1])
+  spec.summary, spec.description, = description.split(/\n\n+/, 3)
+  spec.license = "GPLv3 or later"
+  spec.files = ["README.md", "Rakefile", "Gemfile", "#{spec.name}.gemspec"]
+  spec.files += Dir.glob("lib/**/*.rb")
+  Dir.chdir("bin") do
+    spec.executables = Dir.glob("*")
+  end
+
+  spec.add_runtime_dependency("classifier-reborn")
+  spec.add_runtime_dependency("natto")
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification
+name: tiny-classifier
+version: !ruby/object:Gem::Version
+  version: '1.0'
+platform: ruby
+authors:
+- YUKI "Piro" Hiroshi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-06-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: classifier-reborn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: natto
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- piro.outsider.reflex@gmail.com
+executables:
+- tc-train
+- tc-classify
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- bin/tc-classify
+- bin/tc-train
+- lib/tiny-classifier/base.rb
+- lib/tiny-classifier/classifier.rb
+- lib/tiny-classifier/trainer.rb
+- tiny-classifier.gemspec
+homepage: https://github.com/piroor/tiny-classifier
+licenses:
+- GPLv3 or later
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Command line tool to run text classifier based on naive bayes.
+test_files: []