gibberish_detector 1.0.0

data/gibberish_detector.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'gibberish_detector/version'
+
+Gem::Specification.new do |s|
+  s.name          = "gibberish_detector"
+  s.version       = GibberishDetector::VERSION
+  s.summary       = "Detect gibberish in strings"
+  s.date          = "2014-02-17"
+  s.description   = "This gem uses the Markov chain to study and analyze text and decide if it is gibberish or not."
+  s.authors       = ["Michael Chittenden"]
+  s.email         = ["mchitten@gmail.com"]
+  s.license       = "MIT"
+  s.homepage      = "https://github.com/mchitten/gibberish_detector"
+
+  s.files         = `git ls-files`.split($/)
+  s.test_files    = s.files.grep(%r{test|spec|features})
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "bundler", "~> 1.3"
+  s.add_development_dependency "rake"
+end

data/good.txt

@@ -0,0 +1,6 @@
+rob
+two models
+some long sentence, might suck?
+Project Gutenberg
+a b c
+

data/lib/gibberish_detector.rb

@@ -0,0 +1,139 @@
+require 'yaml'
+require 'gibberish_detector/version'
+require 'string'
+
+class GibberishDetectorException < Exception ; end
+
+class GibberishDetector
+  ACCEPTED_CHARACTERS = 'abcdefghijklmnopqrstuvwxyz ';
+  DATA_FILE = File.join(File.dirname(__FILE__), '..', '.trained_data.yml')
+
+  class << self
+    def gibberish?(text, opts = {})
+      opts[:lib_path] ||= DATA_FILE
+      opts[:raw] ||= false
+
+      raise GibberishDetectorException, "Please run Gibberish.train! to build your trained data file." unless File.exist?(opts[:lib_path])
+
+      trained_library = YAML.load(File.open(opts[:lib_path]))
+      raise GibberishDetectorException, 'Please run Gibberish.train! to build your trained data file.' if trained_library.nil?
+
+      value = _averageTransitionProbability(text, trained_library[:matrix])
+      return value if opts[:raw] == true
+
+      return true if value <= trained_library[:threshold]
+
+      false
+    end
+
+    def train!(opts={})
+      opts[:big_text_file] = 'big.txt'
+      opts[:good_text_file] = 'good.txt'
+      opts[:bad_text_file] = 'bad.txt'
+      opts[:lib_path] = DATA_FILE
+
+      if File.exist?(opts[:big_text_file]) == false || File.exist?(opts[:good_text_file]) == false || File.exist?(opts[:bad_text_file]) == false
+        raise GibberishDetectorException, "We couldn't find one of #{opts[:big_text_file]}, #{opts[:good_text_file]} or #{opts[:bad_text_file]}.  Please ensure all three files exist before training."
+        return false
+      end
+
+      k = ACCEPTED_CHARACTERS.length
+      hsh = {}
+      pos = ACCEPTED_CHARACTERS.dup.split('').each_with_index do |key, index|
+        hsh[key] = index
+      end.reverse
+      pos = hsh
+
+      log_prob_matrix = {}
+      range = (0...k).to_a
+      range.each do |index|
+        arr = {}
+        range.each do |index2|
+          arr[index2] = 10
+        end
+
+        log_prob_matrix[index] = arr
+      end
+
+      lines = File.open(opts[:big_text_file]).read
+      lines.each_line do |line|
+        filtered_line = normalize(line).split('')
+        a = false
+        filtered_line.each do |b|
+          if a != false
+            log_prob_matrix[pos[a]] ||= {}
+            log_prob_matrix[pos[a]][pos[b]] ||= 0
+            log_prob_matrix[pos[a]][pos[b]] += 1
+          end
+          a = b
+        end
+      end
+
+      log_prob_matrix.each do |i, row|
+        s = row.values.inject(:+).to_f
+        row.each do |k, j|
+          log_prob_matrix[i][k] = Math.log(j / s)
+        end
+      end
+
+      good_lines = File.open(opts[:good_text_file]).read
+      good_probs = []
+      good_lines.each_line do |line|
+        good_probs << _averageTransitionProbability(line.chomp, log_prob_matrix)
+      end
+
+      bad_lines = File.open(opts[:bad_text_file]).read
+      bad_probs = []
+      bad_lines.each_line do |line|
+        bad_probs << _averageTransitionProbability(line.chomp, log_prob_matrix)
+      end
+
+      min_good_probs = good_probs.min
+      max_bad_probs = bad_probs.max
+
+      if min_good_probs <= max_bad_probs
+        raise GibberishDetectorException, "The prob counts are invalid."
+      end
+
+      threshold = (min_good_probs + max_bad_probs) / 2
+      File.open(opts[:lib_path], 'w+') do |file|
+        data = {
+          :matrix => log_prob_matrix,
+          :threshold => threshold
+        }
+
+        file << data.to_yaml
+      end
+    end
+
+    private
+      def normalize(text)
+        text.downcase.gsub(/[^a-z\ ]/, '')
+      end
+
+      def _averageTransitionProbability(line, log_prob_matrix)
+        log_prob = 1.0
+        transition_ct = 0
+
+        hsh = {}
+        ACCEPTED_CHARACTERS.dup.split('').each_with_index do |key, index|
+          hsh[key] = index
+        end.reverse
+        pos = hsh
+
+        filtered_line = normalize(line.dup).split('')
+        a = false
+        filtered_line.each do |b|
+          if a != false
+            log_prob += log_prob_matrix[pos[a]][pos[b]]
+            transition_ct += 1
+          end
+
+          a = b
+        end
+
+        Math.exp(log_prob / [transition_ct, 1].max)
+      end
+  end
+end
+

data/lib/gibberish_detector/version.rb

@@ -0,0 +1,3 @@
+class GibberishDetector
+  VERSION = "1.0.0"
+end

data/lib/string.rb

@@ -0,0 +1,5 @@
+class String
+  def gibberish?
+    GibberishDetector.gibberish?(self)
+  end
+end

data/test/test_gibberish_detector.rb

@@ -0,0 +1,13 @@
+require 'test/unit'
+require 'gibberish_detector'
+
+class GibberishDetectorTest < Test::Unit::TestCase
+  def test_gibberish
+    gibberish = "asodfjasdf"
+    assert_equal gibberish.gibberish?, true
+  end
+  def test_non_gibberish
+    gibberish = "hello world"
+    assert_equal gibberish.gibberish?, false
+  end
+end

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification
+name: gibberish_detector
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Michael Chittenden
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem uses the Markov chain to study and analyze text and decide if
+  it is gibberish or not.
+email:
+- mchitten@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .trained_data.yml
+- CREDITS.md
+- Gemfile
+- README.md
+- Rakefile
+- bad.txt
+- big.txt
+- gibberish_detector.gemspec
+- good.txt
+- lib/gibberish_detector.rb
+- lib/gibberish_detector/version.rb
+- lib/string.rb
+- test/test_gibberish_detector.rb
+homepage: https://github.com/mchitten/gibberish_detector
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.5
+signing_key: 
+specification_version: 4
+summary: Detect gibberish in strings
+test_files:
+- gibberish_detector.gemspec
+- test/test_gibberish_detector.rb