bad_word_detector 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+gemspec
+

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Fedotov Daniil
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,56 @@
+# BadWordDetector
+
+Swear word detector
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'bad_word_detector'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install bad_word_detector
+
+## Usage
+
+Detects `#uck` , `F|_|__C_K` and other variations of hidden swear words in text.
+
+Usage:
+
+```ruby
+finder = BadWordDetector.new
+finder.find("What the #uck")
+```
+
+it will return BadWord object
+
+Transformation rules is defined in form: 
+
+```ruby
+{"#" => {"symbol"=>"f", "weight" => 2}} # weight is optional
+```
+
+Or in file conf/rules.yaml 
+
+List of swear words is located in conf/library.yaml
+
+Whitelist of english words in conf/whitelist.yaml
+
+You can also set own rules:
+
+```ruby
+finder = BadWordDetector.new rules, library, whitelist
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+
+Rake::TestTask.new do |t|
+    t.libs << "test"
+    t.test_files = FileList['test/test*.rb']
+    t.verbose = true
+end

data/bad_word_detector.gemspec

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/bad_word_detector/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Fedotov Daniil"]
+  gem.email         = ["fedotov.danil@gmail.com"]
+  gem.summary   = %q{Swear word detector}
+  gem.description       = %q{
+    Detects #uck F|_|__C_K and other variations of hidden swear words in text.
+    Usage:
+    ```
+        finder = BadWordDetector.new
+        finder.find("What the #uck")
+        it will return BadWord object
+    ```
+    Transformation rules is defined in form: {"#" => {"symbol"=>"f", "weight" => 2}} (where weight is optional)
+    in file conf/rules.yaml 
+    List of swear words is located in conf/library.yaml
+    Whitelist of english words in conf/whitelist.yaml
+    You can also set own rules:
+        finder = BadWordDetector.new rules, library, whitelist
+  }
+  gem.homepage      = "https://github.com/hairyhum/bad-words.ruby"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "bad_word_detector"
+  gem.require_paths = ["lib"]
+  gem.version       = BadWordDetector::VERSION
+  gem.add_development_dependency('yard')
+  gem.add_development_dependency('redcarpet')
+end

data/bin/bad_word_detector

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << "#{File.expand_path(File.dirname(__FILE__))}/../lib"
+
+require 'bad_word_detector'
+
+string = ARGV[0]
+
+if string
+    puts BadWordDetector.new().find(ARGV[0])
+else    
+    puts 'Specify input string'
+end

data/lib/bad_word_detector.rb

@@ -0,0 +1,200 @@
+require "yaml"
+require "set"
+require "bad_word_detector/state"
+require "bad_word_detector/prefix_tree"
+require "bad_word_detector/rule"
+require "bad_word_detector/version"
+require "bad_word_detector/whitelist"
+require "bad_word_detector/bad_word"
+
+class BadWordDetector
+
+  # Create new badword checker
+  # 
+  # @param rules [Hash<String,Array<Hash<String, any>>>] Hash where values are arrays 
+  #   of Hash<['symbol', 'weight'], any> where weight is optional
+  # 
+  # @param library [Array<String>] Array of bad words to find
+  # 
+  # @param whitelist [Array<String>] Array of words that is acceptable. Used in false-positive check
+  # 
+  def initialize(rules = nil, library = nil,  whitelist = nil)
+    confdir = File.expand_path(File.dirname(__FILE__) + "/conf")
+    rules ||= YAML.load_file("#{confdir}/rules.yaml")
+    library ||= YAML.load_file("#{confdir}/library.yaml")
+
+    @rule_sets = rules.select do |key, _|
+      key.to_s.length == 1
+    end.hmap do |key, rule|
+      key = key.to_s
+      rule = rule.map do |item|
+        Rule.new(key, item['symbol'], item['weight'])
+      end
+      rule << Rule.new(key, key, 3)
+      [key, rule]
+    end
+
+    @string_sets = rules.select do |key, _|
+      key.to_s.length > 1
+    end.hmap do |key, rule|
+      key = key.to_s
+      rule = rule.map do |item|
+        Rule.new(key, item['symbol'], item['weight'])
+      end
+      [key, rule]
+    end
+
+    @library = PrefixTree.new library
+    @whitelist = Whitelist.new whitelist || YAML.load_file("#{confdir}/whitelist.yaml")
+  end
+
+  #
+  # Searches string for some word in library
+  # 
+  # @param text [String] String to search
+  # 
+  # @param return_white [Boolean] Flag to indicate if search should return whitelist word
+  # 
+  # @return [BadWord, nil] BadWord object, containing information about found word and it's position in text
+  # 
+  def find(text, return_white = false)
+    downcased = text.downcase
+    length = text.length
+    index = 0
+    while index < length
+      found = find_part(downcased, index)
+      if found 
+        word = BadWord.new(
+          found[:word], 
+          text, 
+          index, 
+          found[:length], 
+          @whitelist)
+        if not word.white? or return_white
+          return word
+        end
+      end
+      index += 1
+    end
+  end
+
+private
+  def find_part(text, index)
+    input = text[index..-1]
+    found = unless input.start_with? ' '
+      process(input, @library)
+    end
+    if found
+      word, length = found
+      {:length => length, :word => word}
+    end
+  end
+
+
+  def process(string, library)
+    plain = library[string]
+    if plain && plain.value
+      return plain.value, plain.value.length
+    end
+    passed = []
+    bad_states = []
+    queue = [State.new([], library)]
+    until queue.empty?
+      state = queue.shift
+      new_states = get_new_states state, string
+      if new_states
+        passed << state
+        success_index = new_states.index(&:success?)
+        if success_index
+          new_state = new_states[success_index]
+          return new_state.text, new_state.length
+        else
+          states = (new_states - bad_states) - passed
+          push_states queue, states
+        end
+      else
+        bad_states << state
+      end
+    end
+    nil
+  end
+
+  def push_states(queue, states)
+    states.each do |state|
+      weight = state.weight
+      if queue.any? { |q_state| q_state == state }
+        next
+      end
+      unless weight < 0.3
+        if queue.empty? || weight < queue.last.weight
+          queue << state
+        elsif weight > queue.first.weight
+          queue.insert(0, state)
+        else
+          new_index = queue.length
+          queue.each_with_index do |item, index|
+            if item.weight < weight
+              new_index = index
+            end
+          end
+          queue.insert(new_index, state)
+        end
+      end
+    end
+  end
+
+  def get_new_states(state, string)
+    next_symbols = get_next_symbols state.length, string
+    unless next_symbols
+      return nil
+    end
+    new_states = append_path next_symbols, state
+    if new_states.empty?
+      nil
+    else
+      new_states
+    end
+  end
+
+  def append_path(symbols, state)
+    symbols.map do |sym|
+      state.append sym
+    end.reject do |new_state|
+      new_state.failure?
+    end
+  end
+
+  def get_next_symbols(index, string)
+    char = string[index]
+    if char
+      char = char.to_s
+      get_rules(char, string[index..-1]) || []
+    end
+  end
+
+  def get_rules(char, string)
+    char_rules = @rule_sets[char] || [Rule.self(char)]
+    if char_rules.none? { |rule| rule.symbol == '' }
+      char_rules << Rule.empty(char)
+    end
+
+    string_rules = @string_sets.select do |k, _|
+      k.start_with?(char) and string.start_with?(k)
+    end.values.flatten
+
+    char_rules.concat(string_rules)
+  end
+
+end
+
+class Hash
+  def hmap
+    result = {}
+    self.each do |k, v|
+      k, v = yield k, v
+      result[k] = v
+    end
+    result
+  end
+
+end

data/lib/bad_word_detector/bad_word.rb

@@ -0,0 +1,30 @@
+class BadWord
+  attr_reader :text, :word, :index, :source, :white_words
+
+  #
+  # Create new BadWord
+  # 
+  # @param word [String] found word
+  # @param source [String] Source text where word was found
+  # @param index [Integer] index of word in source
+  # @param length [Integer] word length
+  # @param whitelist [Array<String>] Whitelist words
+  def initialize(word, source, index, length, whitelist)
+    @index = index
+    @length = length
+    @word = word
+    @source = source
+    word_end = @index+@length-1
+    space_location = @source.index(' ', word_end) || 0
+    @text = @source[@index..space_location-1]
+    @white_words = whitelist.check_bad_word(self)
+  end
+
+  #
+  # Check if word is in whitelist
+  # @return [true, false]
+  def white?
+    !!@white_words
+  end
+
+end

data/lib/bad_word_detector/prefix_tree.rb

@@ -0,0 +1,56 @@
+class PrefixTree
+  def initialize(items = [], hash_tree = {})
+    @hash_tree = hash_tree.clone
+    unless items.empty?
+      items.each do |i|
+        self << i
+      end
+    end
+  end
+
+  def hash_tree
+    @hash_tree
+  end
+
+  def << (string)
+    parts = string.chars
+    new_hash = self.hash_tree
+    parts.each do |part|
+      unless new_hash[part]
+        new_hash[part] = {}
+      end
+      new_hash = new_hash[part]
+    end
+    new_hash[:value] = string
+    self
+  end
+
+  def [] (string = '')
+    parts = string.chars
+    new_hash = self.hash_tree
+    parts.each do |part|
+      unless new_hash[part]
+        return nil
+      end
+      new_hash = new_hash[part]
+    end
+    PrefixTree.new [], new_hash
+  end
+
+  def value
+    self.hash_tree[:value]
+  end
+
+  def children? (string)
+    (self[string].hash_tree.keys - [:values]).any?
+  end
+
+  def clone
+    PrefixTree.new [], self.hash_tree
+  end
+
+  def == (sec)
+    sec.hash_tree == self.hash_tree
+  end
+
+end