sensitive_words 0.0.1

data/dictionary/dict2.txt

@@ -0,0 +1,66 @@
+6-4tianwang
+89-64cdjp
+ADMIN
+Administrator
+asshole
+BLOWJOB
+chinaliberal
+chinamz
+chinesenewsnet
+Clockgemstone
+creaders
+Crestbone
+dajiyuan
+dfdz
+DICK
+falun
+falundafa
+Feelmistone
+freechina
+freenet
+fuck
+gcd
+Gruepin
+Guichuideng
+HACKING
+hongzhi
+hrichina
+HUANET
+hypermart.net
+incest
+jiangdongriji
+jiaochuang
+jiaochun
+KEFU
+KISSMYASS
+lihongzhi
+minghui
+minghuinews
+nacb
+Neckromancer
+NMIS
+PAPER64
+penis
+qiangjian
+renminbao
+renmingbao
+SHIT
+SUCKPENIS
+taip
+tibetalk
+triangle
+triangleboy
+Tringel
+UltraSurf
+ustibet
+voachinese
+wangce
+WEBZEN
+wstaiji
+xinsheng
+YUMING
+zangdu
+ZHENGJIAN
+ZHENGJIANWANG
+ZHENSHANREN
+zhuanfalun

data/lib/sensitive_words.rb

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+
+class SensitiveWords
+
+  @@dict = {}
+
+  class << self
+
+    def load_dict(dict_path)
+      new_dict = get_dict_file_hash(dict_path)
+      dict = @@dict.merge new_dict
+      @@dict = dict
+    rescue Errno::ENOENT => boom
+      puts "#{boom.class} - #{boom.message}"
+    end
+    
+    def get_dict_file_hash(path)
+      tree = {}
+      file = File.open(path, 'r')
+      if file
+        file.each_line do |line|
+          line = line.chomp
+          next if line.empty?
+          node = nil
+          line.chars.each do |c|
+            if node
+              node[c] ||= {}
+              node = node[c]
+            else
+              tree[c] ||= {}
+              node = tree[c]
+            end
+          end
+          node[:end] = :id
+        end
+      end
+      tree
+    ensure
+      file.close if file
+    end
+
+    def sensitive_words(input,max=nil)
+      ins = SensitiveWords.new(input)
+      max = max.to_i
+      if max > 0
+        ins.sensitive_words(max)
+      else
+        ins.all_sensitive_words
+      end
+    end
+
+  end
+
+  def initialize(input)
+    @input = input
+    @words = []
+  end
+  
+  #只要有限个敏感词
+  def sensitive_words(max)
+    @node, @words = @@dict, []
+    @word, @queue = '', []
+
+    @input.chars.each do |char|
+      break if @words.size >= max
+      loop do 
+        break if @queue.empty?
+        chr = @queue.shift
+        process_check(chr, true)
+      end
+      process_check(char)
+    end
+
+    process_check('')
+    @words.first(max)
+  end
+
+  #所有的敏感词
+  def all_sensitive_words
+    @node, @words = @@dict, []
+    @word, @queue = '', []
+
+    @input.chars.each do |char|
+      loop do 
+        break if @queue.empty?
+        chr = @queue.shift
+        process_check(chr, true)
+      end
+      process_check(char)
+    end
+
+    process_check('')
+    @words
+  end
+
+  private
+
+  def process_check(char,queuing=false)
+
+    match, word = nil, nil
+
+    if @node[char]
+      @word << char
+      @node = @node[char]
+      match = :id
+    else
+      if @node[:end]
+        word = @word
+      end
+      lth = @word.length
+      if lth > 0
+        if queuing
+          @queue.unshift char
+        else
+          if lth > 1
+            @queue += @word.chars.last(lth-1)
+          end
+          @queue << char
+        end
+      end
+
+      @node = @@dict
+      @word = ''
+    end
+
+    if !match && word
+      @words << word
+    end
+  end
+  
+end

data/sensitive_words.gemspec

@@ -0,0 +1,21 @@
+# coding: utf-8
+$LOAD_PATH.unshift('lib')
+
+version = '0.0.1'
+
+Gem::Specification.new 'sensitive_words',version do |spec|
+  spec.authors       = ["Jeffrey"]
+  spec.email         = ["jeffrey6052@163.com"]
+  spec.description   = "检索文章中的敏感词"
+  spec.summary       = "-"
+  spec.homepage      = "https://github.com/maymay25/sensitive_words"
+  spec.license       = "MIT"
+
+  spec.files         = ['dictionary/dict1.txt',
+                        'dictionary/dict2.txt',
+                        'lib/sensitive_words.rb',
+                        'sensitive_words.gemspec',
+                        'test.rb',
+                        'README.md']
+                        
+end

data/test.rb

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+require 'sensitive_words'
+
+#首先载入敏感词词典
+SensitiveWords.load_dict("#{__dir__}/dictionary/dict1.txt")
+SensitiveWords.load_dict("#{__dir__}/dictionary/dict2.txt") #可以载入多次
+
+article = "习近平周永暴干康BLOWJOBjeffrey哈哈哈流氓政府"
+
+#找出文章中的所有敏感词
+words = SensitiveWords.sensitive_words(article)
+puts words.inspect   # => ["习近平", "暴干", "BLOWJOB", "流氓政府"]
+
+#或者只需要指定数量上限的敏感词
+words = SensitiveWords.sensitive_words(article,2)
+puts words.inspect   # => ["习近平", "暴干"]

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: sensitive_words
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jeffrey
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-11 00:00:00.000000000 Z
+dependencies: []
+description: "检索文章中的敏感词"
+email:
+- jeffrey6052@163.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- dictionary/dict1.txt
+- dictionary/dict2.txt
+- lib/sensitive_words.rb
+- sensitive_words.gemspec
+- test.rb
+- README.md
+homepage: https://github.com/maymay25/sensitive_words
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: "-"
+test_files: []