harmonious_check 0.0.2

data/harmonious_dictionary.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "harmonious_dictionary/version"
+
+Gem::Specification.new do |s|
+  s.name        = "harmonious_check"
+  s.version     = HarmoniousDictionary::VERSION
+  s.authors     = ["amelia9f"]
+  s.email       = ["amelia9fpyjo@gmx.com"]
+  s.homepage    = "https://github.com/amelia9f/harmonious_check"
+  s.summary     = %q{filter any words that need to be harmonized}
+  s.description = %q{和谐宝典用于检查输入是否包含中文或英文敏感词，并可替换为特殊字符。速度比常规的正则匹配要快10倍以上。生活在天朝，和谐宝典必须人手必备。}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec"
+
+end

data/lib/generators/harmonious_dictionary/setup/setup_generator.rb

@@ -0,0 +1,16 @@
+require "rails/generators"
+
+module HarmoniousDictionary
+  module Generators
+    class SetupGenerator < ::Rails::Generators::Base
+      desc "This generator creates necessary at config/harmonious_dictionary"
+      source_root File.expand_path("../templates", __FILE__)
+
+      def generate_setup  
+        #copy_file "remote_server.yml", "config/harmonious_dictionary/remote_server.yml"  
+        copy_file "chinese_dictionary.txt", "config/harmonious_dictionary/chinese_dictionary.txt"  
+        copy_file "english_dictionary.txt", "config/harmonious_dictionary/english_dictionary.txt"  
+      end
+    end
+  end
+end

data/lib/generators/harmonious_dictionary/setup/templates/remote_server.yml

@@ -0,0 +1,8 @@
+development:
+  url: 127.0.0.1:4100
+
+test:
+  url: 127.0.0.1:4100
+
+production:
+  url: 127.0.0.1:4100

data/lib/harmonious_dictionary/app.rb

@@ -0,0 +1,18 @@
+require 'sinatra/base'
+
+module HarmoniousDictionary
+  class App < Sinatra::Base
+    set :root,   File.dirname(__FILE__) + "/.."
+    set :app_file, __FILE__
+
+    post '/segment' do
+      @input = params[:input]
+      @result = HarmoniousDictionary::Rseg.segment(@input).join(' ')
+    end
+
+    post '/seg' do
+      @input = params[:input]
+      @result = HarmoniousDictionary::Rseg.segment(@input)
+    end
+  end
+end

data/lib/harmonious_dictionary/engines/dict.rb

@@ -0,0 +1,51 @@
+module HarmoniousDictionary
+  module RsegEngine
+    class Dict < Engine    
+      def initialize(&block)
+        @dict_path = block.call
+        @word = ''
+        super
+      end
+
+      def dictionary
+        @@root
+      end
+    
+      def process(char)
+        @root ||= load_dict(@dict_path)
+        @node ||= @root
+
+        match = false
+        word = nil
+        
+        if @node[char]
+          @word << char
+          @node = @node[char]
+          match = true
+        else
+          if @node[:end] || @word.chars.to_a.length == 1
+            word = @word
+          else
+            word = @word.chars.to_a
+          end
+        
+          @node = @root
+          @word = ''
+          match = false
+        end
+        [match, word]
+      end
+    
+      private
+
+      def load_dict(path)
+        begin
+          File.open(path, "rb") {|io| Marshal.load(io) }
+        rescue => e
+          puts e
+          exit
+        end
+      end
+    end
+  end
+end

data/lib/harmonious_dictionary/engines/engine.rb

@@ -0,0 +1,21 @@
+module HarmoniousDictionary
+  module RsegEngine
+    class Engine
+      def initialize
+        @running = true
+      end
+    
+      def stop
+        @running = false
+      end
+    
+      def run
+        @running = true
+      end
+    
+      def running?
+        @running
+      end
+    end
+  end
+end

data/lib/harmonious_dictionary/engines/english.rb

@@ -0,0 +1,29 @@
+require 'set'
+
+module HarmoniousDictionary
+  module RsegEngine
+    LETTER_SYMBOLS = Set.new ('a'..'z').to_a + ('A'..'Z').to_a
+
+    class English < Engine
+      def initialize
+        @word = ''
+        super
+      end
+    
+      def process(char)
+        match = false
+        word = nil
+      
+        if LETTER_SYMBOLS.include?(char)
+          @word << char
+          match = true
+        else
+          word = @word
+          @word = ''
+          match = false
+        end
+        [match, word]
+      end  
+    end
+  end
+end

data/lib/harmonious_dictionary/filters/conjunction.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+
+require 'set'
+
+module RsegFilter
+  class Conjunction
+    @@conjunctions = Set.new %W(给 的 说 对 在 和 是 被 最 所 那 由 这 有 将 你 会 与 他 为 不 没 很 了 啊 哦 呵 把 去 从)
+
+    def self.filter(char)
+      @@conjunctions.include?(char) ? :conjunction : char
+    end
+  end
+end

data/lib/harmonious_dictionary/filters/fullwidth.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+
+module RsegFilter
+  class Fullwidth
+    @@fullwidth_chars = {'１' => '1', '２' => '2', '３' => '3', '４' => '4', '５' => '5', '６' => '6', '７' => '7', '８' => '8',
+                       '９' => '9', '０' => '0', 'ａ' => 'a', 'ｂ' => 'b', 'ｃ' => 'c', 'ｄ' => 'd', 'ｅ' => 'e', 'ｆ' => 'f',
+                       'ｇ' => 'g', 'ｈ' => 'h', 'ｉ' => 'i', 'ｊ' => 'j', 'ｋ' => 'k', 'ｌ' => 'l', 'ｍ' => 'm', 'ｎ' => 'n',
+                       'ｏ' => 'o', 'ｐ' => 'p', 'ｑ' => 'q', 'ｒ' => 'r', 'ｓ' => 's', 'ｔ' => 't', 'ｕ' => 'u', 'ｖ' => 'v',
+                       'ｗ' => 'w', 'ｘ' => 'x', 'ｙ' => 'y', 'ｚ' => 'z', 'Ａ' => 'A', 'Ｂ' => 'B', 'Ｃ' => 'C', 'Ｄ' => 'D',
+                       'Ｅ' => 'E', 'Ｆ' => 'F', 'Ｇ' => 'G', 'Ｈ' => 'H', 'Ｉ' => 'I', 'Ｊ' => 'J', 'Ｋ' => 'K', 'Ｌ' => 'L',
+                       'Ｍ' => 'M', 'Ｎ' => 'N', 'Ｏ' => 'O', 'Ｐ' => 'P', 'Ｑ' => 'Q', 'Ｒ' => 'R', 'Ｓ' => 'S', 'Ｔ' => 'T', 
+                       'Ｕ' => 'U', 'Ｖ' => 'V', 'Ｗ' => 'W', 'Ｘ' => 'X', 'Ｙ' => 'Y', 'Ｚ' => 'Z', '－' => '-', '＋' => '+',
+                       '—' => '-', '，' => ',', '／' => '/', '·' => '.'}
+                     
+    class << self
+      def filter(char)
+        @@fullwidth_chars[char].nil? ? char : @@fullwidth_chars[char]
+      end
+    end
+  end
+end

data/lib/harmonious_dictionary/filters/symbol.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+
+require 'set'
+
+module RsegFilter
+  class Symbol
+    @@separators = Set.new ['｀', '［', '］', '、', '＝', '‘', '；', '。', '｜', '？', '》', 
+                  '《', '：', '“', '｛', '｝', '）', '（', '＊', '…', '＃', '！',
+                  '～', '’', '”', '〕', '〈', '〉', '「', '」', '『', '』', '〖', '〗', 
+                  '【', '】', '＜', '＞', '`', '~', '!', '@', '#', '^', 
+                  '&', '*', '\\', '(', ')', '=', '{', '}', '[', ']', 
+                  '|', ';', ':', "'", '<', '>', '?', "\n", "\t", "\r", 
+                  ' ', '-', '/', '+', ',', '　']
+    def self.filter(char)
+      @@separators.include?(char) ? :symbol : char
+    end
+  end
+end

data/lib/harmonious_dictionary/model_additions.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+
+module HarmoniousDictionary
+  module ModelAdditions
+    def validate_harmonious_of(attr_names, option = {})
+      configuration = {message:'不能含有敏感词'}
+      library = option.values[0].to_s
+      configuration.update(attr_names.pop) if attr_names.last.is_a?(Hash)
+      validates_each attr_names do |model, attribute, value|
+        unless value.blank?
+          if option.size == 0
+            model.errors.add(attribute, configuration[:message]) unless HarmoniousDictionary.clean?(value)
+          else
+            model.errors.add(attribute, configuration[:message]) unless HarmoniousDictionary.clean?(value, library)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/harmonious_dictionary/railtie.rb

@@ -0,0 +1,23 @@
+module HarmoniousDictionary
+  class Railtie < Rails::Railtie
+    # config.harmonious_dictionary = ActiveSupport::OrderedOptions.new
+    # config.harmonious_dictionary.use_remote_server = false
+    # config.after_initialize do
+    #   if config.harmonious_dictionary.use_remote_server
+    #     Rseg.instance.load_remote_url_config
+    #   end
+    # end
+    
+    rake_tasks do
+      load "tasks/generate_dictionary.rake"
+    end
+
+    initializer 'HarmoniousDictionary.model_additions' do
+      ActiveSupport.on_load :active_record do
+        extend ModelAdditions
+      end
+    end
+
+
+  end
+end

data/lib/harmonious_dictionary/rseg.rb

@@ -0,0 +1,185 @@
+# encoding: utf-8
+
+require 'singleton'
+require 'net/http'
+require 'yaml'
+
+require File.join(File.dirname(__FILE__), 'engines/engine')
+require File.join(File.dirname(__FILE__), 'engines/dict')
+require File.join(File.dirname(__FILE__), 'engines/english')
+
+require File.join(File.dirname(__FILE__), 'filters/fullwidth')
+require File.join(File.dirname(__FILE__), 'filters/symbol')
+require File.join(File.dirname(__FILE__), 'filters/conjunction')
+
+module HarmoniousDictionary
+  class Rseg
+    include Singleton
+    include RsegEngine
+    include RsegFilter
+    attr_writer :input
+    attr_accessor :model
+    
+    class << self    
+      def segment(input, model)
+        HarmoniousDictionary::Rseg.instance.model = model
+        HarmoniousDictionary::Rseg.instance.input = input
+        HarmoniousDictionary::Rseg.instance.segment
+      end
+      
+      def load(dict)
+        HarmoniousDictionary::Rseg.instance
+        nil
+      end
+      
+      def remote_segment(input)
+        begin
+          response = Net::HTTP.post_form(URI.parse("http://#{HarmoniousDictionary::Rseg.instance.remote_url}/seg"), :input => input)
+          response.code == '200' ? response.body.split(' ') : 
+              ["Can't connect to http://#{HarmoniousDictionary::Rseg.instance.remote_url}/seg\nUse rseg_server to start it"]
+        rescue
+          ["Can't connect to http://#{HarmoniousDictionary::Rseg.instance.remote_url}/seg\nUse rseg_server to start it"]
+        end
+      end
+    end
+
+    def initialize
+      @input = ''
+      @words = [] 
+    end
+
+    def remote_url
+      @remote_url ||= load_remote_url_config
+    end
+    
+    def segment
+      init_operate
+      @words = []
+      @input.chars.each do |origin|
+        char = filter(origin)
+        process(char, origin)
+      end
+      
+      process(:symbol, '')
+      @words
+    end
+
+    private
+
+    def init_operate
+      @chinese_dictionary_path = chinese_dictionary_path
+      init_engines
+      init_filters
+    end
+
+    def filter(char)
+      result = char
+      @filters.each do |klass|
+        result = klass.filter(result)
+      end
+      result
+    end
+    
+    def process(char, origin)
+      nomatch = true
+      word = ''
+      @english_dictionary ||= load_english_dictionary(english_yaml_path)
+      
+      engines.each do |engine|
+        next unless engine.running?
+        match, word = engine.process(char)
+        if match 
+         nomatch = false
+       else
+        word = '' if engine.class == English && !@english_dictionary.include?(word)
+        engine.stop 
+       end
+      end
+      
+      if nomatch
+        if word == ''
+          # 没切出来的就当正常的词，不输出
+          # @words << origin unless char == :symbol 
+          reset_engines
+        else
+          reset_engines
+          @words << word if word.is_a?(String) if word.size >= 2
+          # 我们只需要脏词完全匹配，不需要检查下文
+          # reprocess(word) if word.is_a?(Array)
+          # re-process current char
+          process(char, origin)
+        end
+      end
+    end
+    
+    def reprocess(word)
+      last = word.pop
+
+      word.each do |char|
+        process(char, char)
+      end
+      
+      process(:symbol, :symbol) # 把词加进来
+      process(last, last) # 继续分析词的最后一个字符
+    end
+
+    def reset_engines
+      engines.each do |engine|
+        engine.run
+      end
+    end
+    
+    def engines=(engines)
+      @engines ||= engines
+    end
+
+    def engines
+      @engines
+    end
+
+    def init_filters
+      @filters = [Fullwidth, Symbol]
+    end
+    
+    def init_engines
+      @engines ||= [Dict, English].map do |engine_klass|
+        if engine_klass == Dict
+          engine_klass.new do
+            @dict_path = @chinese_dictionary_path 
+          end
+        else 
+          engine_klass.new
+        end
+      end
+    end
+
+    def load_remote_url_config
+      YAML.load(File.read(File.join(Rails.root, 'config','harmonious_dictionary','remote_server.yml')))[Rails.env]['url']
+    end
+
+    def load_english_dictionary(path)
+      begin
+        YAML.load(File.read(path))
+      rescue => e
+        puts e
+        exit
+      end
+    end
+
+    def english_yaml_path
+      if model.nil?
+        File.join(Rails.root, 'config','harmonious_dictionary','harmonious_english.yml')
+      else
+        File.join(Rails.root, 'config','harmonious_dictionary',"#{model}_harmonious_english.yml")
+      end
+    end
+
+    def chinese_dictionary_path
+      if model.nil?
+        File.join(Rails.root, 'config','harmonious_dictionary','harmonious.hash')
+      else
+        File.join(Rails.root, 'config','harmonious_dictionary',"#{model}_harmonious.hash")
+      end
+    end
+  end
+end

data/lib/harmonious_dictionary/version.rb

@@ -0,0 +1,3 @@
+module HarmoniousDictionary
+  VERSION = "0.0.2"
+end

data/lib/harmonious_dictionary.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+
+require "harmonious_dictionary/rseg"
+require "harmonious_dictionary/version"
+require "harmonious_dictionary/model_additions"
+require "harmonious_dictionary/railtie" if defined? Rails
+   
+
+module HarmoniousDictionary
+  def self.clean?(input, model = nil)
+    results = HarmoniousDictionary::Rseg.segment(input, model)
+    results.size > 0 ? false : true
+  end
+
+  def self.clean_by_remote?(input)
+    results = HarmoniousDictionary::Rseg.remote_segment(input)
+    results.size > 0 ? false : true
+  end
+
+  def self.clean_by_remote(input)
+    results = HarmoniousDictionary::Rseg.remote_segment(input)
+    results.each do |result|
+      encode_result = result.force_encoding('utf-8')
+      input.gsub! /#{encode_result}/,self.clean_word_basic(encode_result)
+    end
+    input
+  end
+
+  def self.clean(input, model = nil)
+    results = HarmoniousDictionary::Rseg.segment(input, model)
+    results.each{|result| input.gsub! /#{result}/,self.clean_word_basic(result) }
+    input
+  end
+
+  def self.harmonious_words(input, model = nil)
+    return HarmoniousDictionary::Rseg.segment(input, model)
+  end
+
+  def self.clean_word_basic(word)
+    clearn_words = ""
+    word.size.times{  clearn_words << "*" }
+    clearn_words
+  end
+
+  def self.chinese_harmonious
+    Rseg.instance.send(:engines).first.dictionary
+  end
+end

data/lib/tasks/generate_dictionary.rake

@@ -0,0 +1,63 @@
+require 'yaml'
+
+namespace :harmonious_dictionary do
+  desc "generate harmonious dictionary for use"
+  task :generate => :environment do
+    chinese_dictionary_path = File.join(Rails.root, 'config','harmonious_dictionary','chinese_dictionary.txt')
+    english_dictionary_path = File.join(Rails.root, 'config','harmonious_dictionary','english_dictionary.txt')
+
+    puts "Processing chinese words..."
+    tree = {}
+    model = ENV['model']
+    process(chinese_dictionary_path, tree)
+    File.open(hash_path(model), "wb") {|io| Marshal.dump(tree, io)}  
+    puts 'Done'
+
+    puts 'Processing english words...'
+    english_dictionary = []
+    process_english_words(english_dictionary_path,english_dictionary)
+    File.open(yaml_path, "wb") {|io| YAML::dump(english_dictionary, io)} 
+    puts 'Done'
+  end
+end
+
+def process_english_words(path,list)
+  File.open(path, 'r') do |file|
+    file.each_line{|line| list << line.gsub!("\n",'') }
+  end
+end
+
+def process(path, tree)
+  File.open(path, 'r') do |file|
+    file.each_line do |line|
+      node = nil
+      line.chars.each do |c|
+        next if c == "\n" || c == "\r"
+        if node
+          node[c] ||= {}
+          node = node[c]
+        else
+          tree[c] ||= Hash.new
+          node = tree[c]
+        end
+      end
+      node[:end] = true
+    end
+  end
+end
+
+def hash_path(model = nil)
+  if model
+    File.join(Rails.root, 'config','harmonious_dictionary',"#{model}_harmonious.hash")
+  else
+    File.join(Rails.root, 'config','harmonious_dictionary','harmonious.hash')
+  end
+end
+
+def yaml_path(model = nil)
+  if model
+    File.join(Rails.root, 'config','harmonious_dictionary',"#{model}_harmonious_english.yml")
+  else 
+    File.join(Rails.root, 'config','harmonious_dictionary','harmonious_english.yml')
+  end
+end

data/spec/harmonious_dictionary_spec.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+
+require 'spec_helper'
+
+describe HarmoniousDictionary do
+
+  describe 'local' do
+    describe 'segment' do
+      it 'should return harmonious word for sentence' do
+        HarmoniousDictionary.harmonious_words('戴秉国在中国').should == ['戴秉国']
+      end
+
+      it 'should return harmonious word for sentence under have model param' do
+        HarmoniousDictionary.harmonious_words('戴秉国在中国', 'user').should == ['戴秉国']
+      end
+
+      it 'should return english,url and chinese words' do
+        HarmoniousDictionary.harmonious_words('戴秉国 in china,watch cctv.com.let fuck it','user').should == ['戴秉国','fuck']
+      end
+
+      it 'should return english  words under have model param' do
+        HarmoniousDictionary.harmonious_words('Gruepin','user').should == ['Gruepin']
+      end
+
+    end
+
+    it 'should find harmonious chinese  words' do
+      HarmoniousDictionary.clean?('李源潮在中国').should == false
+    end
+
+    it 'should find harmonious chinese words under have model param' do
+      HarmoniousDictionary.clean?('李源潮在中国', 'user').should == false
+    end
+
+    it 'should pass good words' do
+      HarmoniousDictionary.clean?('过去').should == true
+    end
+
+    it 'should pass good words under have model param' do
+      HarmoniousDictionary.clean?('过去', 'user').should == true
+    end
+
+    it 'should clean sentence by replace harmonious words by *' do
+      HarmoniousDictionary.clean('戴秉国在中国').should == '***在中国'
+    end
+
+    it 'should clean sentence by replace harmonious words by * under model param' do
+      HarmoniousDictionary.clean('戴秉国在中国', 'user').should == '***在中国'
+    end
+
+    it 'should replace harmonious with *' do
+      HarmoniousDictionary.clean_word_basic('大米').should == '**'
+    end
+  end
+
+  # describe 'use remote' do
+  #   it 'should use remote server for segment' do
+  #     HarmoniousDictionary.clean_by_remote('戴秉国在中国').should == '***在中国'
+  #   end
+  # end
+end