harmonious_dictionary 0.0.1

data/lib/harmonious_dictionary/version.rb

@@ -0,0 +1,3 @@
+module HarmoniousDictionary
+  VERSION = "0.0.1"
+end

data/lib/tasks/generate_dictionary.rake

@@ -0,0 +1,55 @@
+require 'yaml'
+
+namespace :harmonious_dictionary do
+  desc "generate harmonious dictionary for use"
+  task :generate => :environment do
+    chinese_dictionary_path = File.join(Rails.root, 'config','harmonious_dictionary','chinese_dictionary.txt')
+    english_dictionary_path = File.join(Rails.root, 'config','harmonious_dictionary','english_dictionary.txt')
+
+    puts "Processing chinese words..."
+    tree = {}
+    process(chinese_dictionary_path, tree)
+    File.open(hash_path, "wb") {|io| Marshal.dump(tree, io)}  
+    puts 'Done'
+
+    puts 'Processing english words...'
+    english_dictionary = []
+    process_english_words(english_dictionary_path,english_dictionary)
+    File.open(yaml_path, "wb") {|io| YAML::dump(english_dictionary, io)} 
+    puts 'Done'
+  end
+end
+
+def process_english_words(path,list)
+  File.open(path, 'r') do |file|
+    file.each_line{|line| list << line.gsub!("\n",'') }
+  end
+end
+
+def process(path, tree)
+  File.open(path, 'r') do |file|
+    file.each_line do |line|
+      node = nil
+      line.chars.each do |c|
+        next if c == "\n" || c == "\r"
+        if node
+          node[c] ||= {}
+          node = node[c]
+        else
+          tree[c] ||= Hash.new
+          node = tree[c]
+        end
+      end
+      node[:end] = true
+    end
+  end
+end
+
+def hash_path
+  File.join(Rails.root, 'config','harmonious_dictionary','harmonious.hash')
+end
+
+def yaml_path
+  File.join(Rails.root, 'config','harmonious_dictionary','harmonious_english.yml')
+end
+

data/spec/harmonious_dictionary_spec.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+
+require 'spec_helper'
+
+describe HarmoniousDictionary do
+
+  describe 'local' do
+    describe 'segment' do
+      it 'should return harmonious word for sentence' do
+        HarmoniousDictionary.harmonious_words('戴秉国在中国').should == ['戴秉国']
+      end
+
+      it 'should return english,url and chiese words' do
+        HarmoniousDictionary.harmonious_words('戴秉国 in china,watch cctv.com.let fuck it').should == ['戴秉国','fuck']
+      end
+    end
+
+    it 'should find harmonious chinese  words' do
+      HarmoniousDictionary.clean?('李源潮在中国').should == false
+    end
+
+    it 'should pass good words' do
+      HarmoniousDictionary.clean?('过去').should == true
+    end
+
+    it 'should clean sentence by replace harmonious words by *' do
+      HarmoniousDictionary.clean('戴秉国在中国').should == '***在中国'
+    end
+
+    it 'should replace harmonious with *' do
+      HarmoniousDictionary.clean_word_basic('大米').should == '**'
+    end
+  end
+
+  # describe 'use remote' do
+  #   it 'should use remote server for segment' do
+  #     HarmoniousDictionary.clean_by_remote('戴秉国在中国').should == '***在中国'
+  #   end
+  # end
+end

data/spec/model_additions_spec.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+require 'spec_helper'
+
+ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => ":memory:")
+
+ActiveRecord::Schema.define(:version => 1) do
+  create_table :posts do |t|
+    t.string :title
+    t.text   :body
+    t.text   :note
+  end
+end
+
+class Post < ActiveRecord::Base
+  extend HarmoniousDictionary::ModelAdditions
+  validate_harmonious_of :title,:body
+end
+
+describe HarmoniousDictionary::ModelAdditions do
+  let(:post) { @post = Post.create title:'戴秉国在中国',body:'戴秉国在中国',note:'戴秉国在中国' }
+
+  describe 'use local' do
+    it 'should validate for harmonious' do
+      post.errors[:title].should == ['不能含有敏感词']
+    end
+
+    it 'should have error on title' do
+      post.errors[:body].should == ['不能含有敏感词']
+    end
+
+    it 'should allow empty input value' do
+      p = Post.create body:'戴秉国在中国',note:'戴秉国在中国'
+      post.errors[:body].should == ['不能含有敏感词']
+    end
+
+    it 'should filter! any harmonious words' do
+      HarmoniousDictionary.clean(post.body).should == '***在中国'
+    end
+  end
+
+  # 以后再开放
+  # describe 'use remote' do
+  #   before(:each) do
+  #     configuration = double("configuration")
+  #     @double_harmonious_dictionary = double('harmonious_dictionary')
+  #     configuration.stub(:harmonious_dictionary){ @double_harmonious_dictionary }
+  #     @double_harmonious_dictionary.stub(:use_remote_server){ true }
+  #     Rails.stub(:configuration){configuration}
+  #   end
+
+  #   it 'should validate for harmonious' do
+  #     HarmoniousDictionary.should_receive(:clean_by_remote?)
+  #     post.errors[:title].should == ['不能含有敏感词']
+  #   end    
+  # end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,21 @@
+require 'rubygems'
+
+require 'rails'
+require 'active_model'
+require 'active_record'
+require 'harmonious_dictionary'
+
+RSpec.configure do |config|
+  config.color_enabled = true
+  config.formatter     = 'documentation'
+end
+
+module Rails
+  def self.root
+    File.join File.dirname(__FILE__),'../'
+  end
+
+  def self.env
+    'test'
+  end
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: harmonious_dictionary
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Stephen Kong
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-12-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 和谐宝典用于检查输入是否包含中文或英文敏感词，并可替换为特殊字符。速度比常规的正则匹配要快10倍以上。生活在天朝，和谐宝典必须人手必备。
+email:
+- wear63659220@gmail.com
+executables:
+- harmonious_rseg
+- harmonious_server
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- CHANGELOG
+- MIT-LICENSE
+- README.markdown
+- Rakefile
+- benchmark/benchmark.rb
+- benchmark/text_test_100.txt
+- benchmark/text_test_1000.txt
+- benchmark/text_test_10000.txt
+- bin/harmonious_rseg
+- bin/harmonious_server
+- harmonious_dictionary.gemspec
+- lib/generators/harmonious_dictionary/setup/setup_generator.rb
+- lib/generators/harmonious_dictionary/setup/templates/chinese_dictionary.txt
+- lib/generators/harmonious_dictionary/setup/templates/english_dictionary.txt
+- lib/generators/harmonious_dictionary/setup/templates/remote_server.yml
+- lib/harmonious_dictionary.rb
+- lib/harmonious_dictionary/app.rb
+- lib/harmonious_dictionary/engines/dict.rb
+- lib/harmonious_dictionary/engines/engine.rb
+- lib/harmonious_dictionary/engines/english.rb
+- lib/harmonious_dictionary/filters/conjunction.rb
+- lib/harmonious_dictionary/filters/fullwidth.rb
+- lib/harmonious_dictionary/filters/symbol.rb
+- lib/harmonious_dictionary/model_additions.rb
+- lib/harmonious_dictionary/railtie.rb
+- lib/harmonious_dictionary/rseg.rb
+- lib/harmonious_dictionary/version.rb
+- lib/tasks/generate_dictionary.rake
+- spec/harmonious_dictionary_spec.rb
+- spec/model_additions_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/wear/harmonious_dictionary
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: filter any words that need to be harmonized
+test_files:
+- spec/harmonious_dictionary_spec.rb
+- spec/model_additions_spec.rb
+- spec/spec_helper.rb