filter_word 0.0.2

data/lib/generators/filter_word/setup/templates/english_dictionary.txt

@@ -0,0 +1,66 @@
+6-4tianwang
+89-64cdjp
+ADMIN
+Administrator
+asshole
+BLOWJOB
+chinaliberal
+chinamz
+chinesenewsnet
+Clockgemstone
+creaders
+Crestbone
+dajiyuan
+dfdz
+DICK
+falun
+falundafa
+Feelmistone
+freechina
+freenet
+fuck
+gcd
+Gruepin
+Guichuideng
+HACKING
+hongzhi
+hrichina
+HUANET
+hypermart.net
+incest
+jiangdongriji
+jiaochuang
+jiaochun
+KEFU
+KISSMYASS
+lihongzhi
+minghui
+minghuinews
+nacb
+Neckromancer
+NMIS
+PAPER64
+penis
+qiangjian
+renminbao
+renmingbao
+SHIT
+SUCKPENIS
+taip
+tibetalk
+triangle
+triangleboy
+Tringel
+UltraSurf
+ustibet
+voachinese
+wangce
+WEBZEN
+wstaiji
+xinsheng
+YUMING
+zangdu
+ZHENGJIAN
+ZHENGJIANWANG
+ZHENSHANREN
+zhuanfalun

data/lib/tasks/generate_dictionary.rake

@@ -0,0 +1,67 @@
+require 'yaml'
+
+namespace :filter_word do
+  desc "generate harmonious dictionary for use"
+  task :generate => :environment do
+
+    puts "Processing chinese words..."
+    tree = {}
+    model = ENV['model']
+
+    chinese_dictionary = model.nil? ? 'chinese_dictionary.txt' : "#{model}_chinese_dictionary.txt"
+    english_dictionary = model.nil? ? 'english_dictionary.txt' : "#{model}_english_dictionary.txt"
+    chinese_dictionary_path = File.join(Rails.root, 'config','filter_word', chinese_dictionary)
+    english_dictionary_path = File.join(Rails.root, 'config','filter_word', english_dictionary)
+
+    process(chinese_dictionary_path, tree)
+    File.open(hash_path(model), "wb") {|io| Marshal.dump(tree, io)}  
+    puts 'chinese_dictionary hash Done'
+
+    puts 'Processing english words...'
+    english_dictionary_list = []
+    process_english_words(english_dictionary_path, english_dictionary_list)
+    File.open(yaml_path(model), "wb") {|io| YAML::dump(english_dictionary_list, io)} 
+    puts 'english_dictionary yaml Done'
+  end
+end
+
+def process_english_words(path,list)
+  File.open(path, 'r') do |file|
+    file.each_line{|line| list << line.gsub!("\n",'') }
+  end
+end
+
+def process(path, tree)
+  File.open(path, 'r') do |file|
+    file.each_line do |line|
+      node = nil
+      line.chars.each do |c|
+        next if c == "\n" || c == "\r"
+        if node
+          node[c] ||= {}
+          node = node[c]
+        else
+          tree[c] ||= Hash.new
+          node = tree[c]
+        end
+      end
+      node[:end] = true
+    end
+  end
+end
+
+def hash_path(model = nil)
+  if model
+    File.join(Rails.root, 'config','filter_word',"#{model}_harmonious.hash")
+  else
+    File.join(Rails.root, 'config','filter_word','harmonious.hash')
+  end
+end
+
+def yaml_path(model = nil)
+  if model
+    File.join(Rails.root, 'config','filter_word',"#{model}_harmonious_english.yml")
+  else 
+    File.join(Rails.root, 'config','filter_word','harmonious_english.yml')
+  end
+end

data/spec/filter_word_spec.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+
+require 'spec_helper'
+
+describe FilterWord do
+
+  describe 'local' do
+    describe 'segment' do
+      it 'should return harmonious word for sentence' do
+        FilterWord.harmonious_words('戴秉国在中国').should == ['戴秉国']
+      end
+
+      it 'should return harmonious word for sentence under have model param' do
+        FilterWord.harmonious_words('戴秉国在中国', 'user').should == ['戴秉国']
+      end
+
+      it 'should return english,url and chinese words' do
+        FilterWord.harmonious_words('戴秉国 in china,watch cctv.com.let fuck it','user').should == ['戴秉国','fuck']
+      end
+
+      it 'should return english  words under have model param' do
+        FilterWord.harmonious_words('Gruepin','user').should == ['Gruepin']
+      end
+
+    end
+
+    it 'should find harmonious chinese  words' do
+      FilterWord.clean?('李源潮在中国').should == false
+    end
+
+    it 'should find harmonious chinese words under have model param' do
+      FilterWord.clean?('李源潮在中国', 'user').should == false
+    end
+
+    it 'should pass good words' do
+      FilterWord.clean?('过去').should == true
+    end
+
+    it 'should pass good words under have model param' do
+      FilterWord.clean?('过去', 'user').should == true
+    end
+
+    it 'should clean sentence by replace harmonious words by *' do
+      FilterWord.clean('戴秉国在中国').should == '***在中国'
+    end
+
+    it 'should clean sentence by replace harmonious words by * under model param' do
+      FilterWord.clean('戴秉国在中国', 'user').should == '***在中国'
+    end
+
+    it 'should replace harmonious with *' do
+      FilterWord.clean_word_basic('大米').should == '**'
+    end
+  end
+
+end

data/spec/model_additions_spec.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+require 'spec_helper'
+
+ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => ":memory:")
+
+ActiveRecord::Schema.define(:version => 1) do
+  create_table :posts do |t|
+    t.string :title
+    t.text   :body
+    t.text   :note
+  end
+end
+
+class Post < ActiveRecord::Base
+  extend FilterWord::ModelAdditions
+  validate_harmonious_of [:title,:body], model: :post
+end
+
+describe FilterWord::ModelAdditions do
+  let(:post) { @post = Post.create title:'戴秉国在中国',body:'戴秉国在中国',note:'戴秉国在中国' }
+
+  describe 'use local' do
+    it 'should validate for harmonious' do
+      post.errors[:title].should == ['不能含有敏感词']
+    end
+
+    it 'should have error on title' do
+      post.errors[:body].should == ['不能含有敏感词']
+    end
+
+    it 'should allow empty input value' do
+      p = Post.create body:'戴秉国在中国',note:'戴秉国在中国'
+      post.errors[:body].should == ['不能含有敏感词']
+    end
+
+    it 'should filter! any harmonious words' do
+      FilterWord.clean(post.body).should == '***在中国'
+    end
+  end
+
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,21 @@
+require 'rubygems'
+
+require 'rails'
+require 'active_model'
+require 'active_record'
+require 'filter_word'
+
+RSpec.configure do |config|
+  config.color_enabled = true
+  config.formatter     = 'documentation'
+end
+
+module Rails
+  def self.root
+    File.join File.dirname(__FILE__),'../'
+  end
+
+  def self.env
+    'test'
+  end
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: filter_word
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- xiaobo
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: "和谐检测用于检查输入是否包含中文或英文敏感词."
+email:
+- peterwillcn@gmail.com
+executables:
+- harmonious_rseg
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CHANGELOG
+- MIT-LICENSE
+- README.markdown
+- Rakefile
+- benchmark/benchmark.rb
+- benchmark/text_test_100.txt
+- benchmark/text_test_1000.txt
+- benchmark/text_test_10000.txt
+- bin/harmonious_rseg
+- filter_word.gemspec
+- lib/filter_word.rb
+- lib/filter_word/app.rb
+- lib/filter_word/engines/dict.rb
+- lib/filter_word/engines/engine.rb
+- lib/filter_word/engines/english.rb
+- lib/filter_word/filters/conjunction.rb
+- lib/filter_word/filters/fullwidth.rb
+- lib/filter_word/filters/symbol.rb
+- lib/filter_word/model_additions.rb
+- lib/filter_word/railtie.rb
+- lib/filter_word/rseg.rb
+- lib/filter_word/version.rb
+- lib/generators/filter_word/setup/setup_generator.rb
+- lib/generators/filter_word/setup/templates/chinese_dictionary.txt
+- lib/generators/filter_word/setup/templates/english_dictionary.txt
+- lib/tasks/generate_dictionary.rake
+- spec/filter_word_spec.rb
+- spec/model_additions_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/tian-xiaobo/filter_word
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: filter any words that need to be harmonized
+test_files:
+- spec/filter_word_spec.rb
+- spec/model_additions_spec.rb
+- spec/spec_helper.rb