despamilator 0.2

data/History.txt

@@ -0,0 +1,4 @@
+=== 0.0.1 2010-01-30
+
+* 1 major enhancement:
+  * Initial release

data/Manifest.txt

@@ -0,0 +1,44 @@
+History.txt
+Manifest.txt
+PostInstall.txt
+README.rdoc
+Rakefile
+despamilator.gemspec
+lib/despamilator.rb
+lib/despamilator/filter.rb
+lib/despamilator/filter/html_tags.rb
+lib/despamilator/filter/naughty_q.rb
+lib/despamilator/filter/numbers_and_words.rb
+lib/despamilator/filter/script_tag.rb
+lib/despamilator/filter_base.rb
+pkg/despamilator-0.1.gem
+pkg/despamilator-0.1.tgz
+pkg/despamilator-0.1/History.txt
+pkg/despamilator-0.1/Manifest.txt
+pkg/despamilator-0.1/PostInstall.txt
+pkg/despamilator-0.1/README.rdoc
+pkg/despamilator-0.1/Rakefile
+pkg/despamilator-0.1/despamilator.gemspec
+pkg/despamilator-0.1/lib/despamilator.rb
+pkg/despamilator-0.1/lib/despamilator/filter.rb
+pkg/despamilator-0.1/lib/despamilator/filter/html_tags.rb
+pkg/despamilator-0.1/lib/despamilator/filter/naughty_q.rb
+pkg/despamilator-0.1/lib/despamilator/filter/numbers_and_words.rb
+pkg/despamilator-0.1/lib/despamilator/filter/script_tag.rb
+pkg/despamilator-0.1/lib/despamilator/filter_base.rb
+pkg/despamilator-0.1/spec/despamilator_spec.rb
+pkg/despamilator-0.1/spec/filters/html_tags_spec.rb
+pkg/despamilator-0.1/spec/filters/naughty_q_spec.rb
+pkg/despamilator-0.1/spec/filters/numbers_and_words_spec.rb
+pkg/despamilator-0.1/spec/filters/script_tag_spec.rb
+pkg/despamilator-0.1/spec/spec.opts
+pkg/despamilator-0.1/spec/spec_helper.rb
+pkg/despamilator-0.1/tasks/rspec.rake
+spec/despamilator_spec.rb
+spec/filters/html_tags_spec.rb
+spec/filters/naughty_q_spec.rb
+spec/filters/numbers_and_words_spec.rb
+spec/filters/script_tag_spec.rb
+spec/spec.opts
+spec/spec_helper.rb
+tasks/rspec.rake

data/PostInstall.txt

@@ -0,0 +1 @@
+For more information on despamilator or to contribute more filters, see http://github.com/moowahaha/despamliator

data/README.rdoc

@@ -0,0 +1,107 @@
+= Despamilator
+
+* http://github.com/moowahaha/despamliator
+
+== DESCRIPTION:
+
+Despamilator is a plugin based spam detector designed for use on your web forms borne out of two annoyances:
+Spam being submitted in my web forms AND CAPTCHAS being intrusive. Despamilator will apply
+some commonly used heuristics from the world of anti-spam to help you decide whether your users are human or machine.
+
+== FEATURES/PROBLEMS:
+
+* rspec will be run twice. How annoying?!
+
+== SYNOPSIS:
+
+  # using Despamilator
+  dspam = Despamilator.new('some text with an <h2> tag qthhg')
+
+  dspam.score #=> the total score for this string (1 is considered high)
+  dspam.matched_by #=> array of matching filters
+  first_match = dspam.matched_by.first #=> first matching filter
+  first_match.name #=> some string with the name of the filter
+  first_match.description #=> some string to describe
+  first_match.score #=> the individual score assigned by this filter
+
+  # adding a new filter! example: detecting the letter "a"
+  # put the following code in lib/despamilator/filter/detect_letter_a.rb
+  def name
+    'Detecting the letter A'
+  end
+
+  def description
+    'Detects the letter "a" in a string for no reason other than a demo'
+  end
+
+  def parse
+    if self.text.downcase.scan(/a/)
+      # add 0.1 to the score of the text
+      self.append_score = 0.1
+    end
+  end
+
+== FILTERING:
+
+As stated, this is a heuristic scanner so its up to the user to decide the thresholds of the scanner. I usually
+say "it's spam" if the score reaches 1.
+
+The score will be added to incrementally by each matching filter. So if there is some HTML in there, it will be added
+to the score. If there is also a script tag of some sort, that will add more.
+
+Each filter decides how much of a score it assigns. For example, detecting a number next to a letter
+(the numbers_an_words filter) is only a mild hint compared with a script tag (detected by the script_tag filter).
+
+=== NEW FILTERS:
+
+I absolutely welcome new filters and experiments. New filters should be put in the 'lib/despamilator/filter/' directory.
+The core filtering code will detect and use what is in there so you only need to drop the code in.
+Filters should be simple, no classes etc wrapped around them and should try to perform one simple task.
+They should always supply the following methods:
+
+ * name #=> the name of your filter.
+ * description #=> what your filter will look for.
+ * parse #=> the method that will be called when parsing.
+
+Along side the above, the following methods are made available to each filter:
+
+ * text #=> a copy of the text your parser will parse
+ * append_score= #=> method to append a score to the text if there are matches in your parser.
+ * matched? #=> whether or not any filter has so far detected something suspect
+ * score #=> the current score assigned to the text
+
+spec tests are an absolute must!
+
+
+== REQUIREMENTS:
+
+* hoe
+* rspec
+
+
+== INSTALL:
+
+ $ sudo gem install despamilator
+
+== LICENSE:
+
+Copyright (c) 2010 Stephen Hardisty
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,33 @@
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require 'fileutils'
+require './lib/despamilator'
+
+Hoe.plugin :newgem
+# Hoe.plugin :website
+# Hoe.plugin :cucumberfeatures
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.spec 'despamilator' do
+  self.developer 'Stephen Hardisty', 'moowahaha@hotmail.com'
+  self.post_install_message = 'PostInstall.txt'
+  self.rubyforge_name       = self.name # TODO this is default value
+  # self.extra_deps         = [['activesupport','>= 2.0.2']]
+
+end
+
+require 'newgem/tasks'
+Dir['tasks/**/*.rake'].each { |t| load t }
+
+# TODO - want other tests/tasks run by default? Add them to the list
+# remove_task :default
+task :test => [:spec]
+task :default => [:test]
+task :install => [:install_gem]
+
+task :cultivate do
+  system "touch Manifest.txt; rake check_manifest | grep -v \"(in \" | patch"
+  system "rake debug_gem | grep -v \"(in \" > `basename \\`pwd\\``.gemspec"
+end

data/despamilator.gemspec

@@ -0,0 +1,42 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{despamilator}
+  s.version = "0.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Stephen Hardisty"]
+  s.date = %q{2010-03-28}
+  s.description = %q{Despamilator is a plugin based spam detector designed for use on your web forms borne out of two annoyances:
+Spam being submitted in my web forms AND CAPTCHAS being intrusive. Despamilator will apply
+some commonly used heuristics from the world of anti-spam to help you decide whether your users are human or machine.}
+  s.email = ["moowahaha@hotmail.com"]
+  s.extra_rdoc_files = ["History.txt", "Manifest.txt", "PostInstall.txt", "pkg/despamilator-0.1/History.txt", "pkg/despamilator-0.1/Manifest.txt", "pkg/despamilator-0.1/PostInstall.txt"]
+  s.files = ["History.txt", "Manifest.txt", "PostInstall.txt", "README.rdoc", "Rakefile", "despamilator.gemspec", "lib/despamilator.rb", "lib/despamilator/filter.rb", "lib/despamilator/filter/html_tags.rb", "lib/despamilator/filter/naughty_q.rb", "lib/despamilator/filter/numbers_and_words.rb", "lib/despamilator/filter/script_tag.rb", "lib/despamilator/filter_base.rb", "pkg/despamilator-0.1.gem", "pkg/despamilator-0.1.tgz", "pkg/despamilator-0.1/History.txt", "pkg/despamilator-0.1/Manifest.txt", "pkg/despamilator-0.1/PostInstall.txt", "pkg/despamilator-0.1/README.rdoc", "pkg/despamilator-0.1/Rakefile", "pkg/despamilator-0.1/despamilator.gemspec", "pkg/despamilator-0.1/lib/despamilator.rb", "pkg/despamilator-0.1/lib/despamilator/filter.rb", "pkg/despamilator-0.1/lib/despamilator/filter/html_tags.rb", "pkg/despamilator-0.1/lib/despamilator/filter/naughty_q.rb", "pkg/despamilator-0.1/lib/despamilator/filter/numbers_and_words.rb", "pkg/despamilator-0.1/lib/despamilator/filter/script_tag.rb", "pkg/despamilator-0.1/lib/despamilator/filter_base.rb", "pkg/despamilator-0.1/spec/despamilator_spec.rb", "pkg/despamilator-0.1/spec/filters/html_tags_spec.rb", "pkg/despamilator-0.1/spec/filters/naughty_q_spec.rb", "pkg/despamilator-0.1/spec/filters/numbers_and_words_spec.rb", "pkg/despamilator-0.1/spec/filters/script_tag_spec.rb", "pkg/despamilator-0.1/spec/spec.opts", "pkg/despamilator-0.1/spec/spec_helper.rb", "pkg/despamilator-0.1/tasks/rspec.rake", "spec/despamilator_spec.rb", "spec/filters/html_tags_spec.rb", "spec/filters/naughty_q_spec.rb", "spec/filters/numbers_and_words_spec.rb", "spec/filters/script_tag_spec.rb", "spec/spec.opts", "spec/spec_helper.rb", "tasks/rspec.rake"]
+  s.homepage = %q{http://github.com/moowahaha/despamliator}
+  s.post_install_message = %q{PostInstall.txt}
+  s.rdoc_options = ["--main", "README.rdoc"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{despamilator}
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{Despamilator is a plugin based spam detector designed for use on your web forms borne out of two annoyances: Spam being submitted in my web forms AND CAPTCHAS being intrusive}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<rubyforge>, [">= 2.0.3"])
+      s.add_development_dependency(%q<gemcutter>, [">= 0.5.0"])
+      s.add_development_dependency(%q<hoe>, [">= 2.5.0"])
+    else
+      s.add_dependency(%q<rubyforge>, [">= 2.0.3"])
+      s.add_dependency(%q<gemcutter>, [">= 0.5.0"])
+      s.add_dependency(%q<hoe>, [">= 2.5.0"])
+    end
+  else
+    s.add_dependency(%q<rubyforge>, [">= 2.0.3"])
+    s.add_dependency(%q<gemcutter>, [">= 0.5.0"])
+    s.add_dependency(%q<hoe>, [">= 2.5.0"])
+  end
+end

data/lib/despamilator/filter/html_tags.rb

@@ -0,0 +1,116 @@
+def parse
+  html = self.text.downcase
+  
+  html_tags.each do |tag|
+    if html.match(/<\s*#{tag}\W/) || html.match(/<\n*#{tag}\W/) || html.match(/\W#{tag}\s*\//) || html.match(/\W#{tag}\n*\//)
+      self.append_score = 0.3
+    end
+  end
+end
+
+def name
+  'Detects HTML tags in text'
+end
+
+def description
+  'Searches for various HTML tags'
+end
+
+def html_tags
+  # make sure these are lowercase, in order to save processing
+  [
+          '!--',
+          '!doctype',
+          'a',
+          'abbr',
+          'acronym',
+          'address',
+          'applet',
+          'area',
+          'b',
+          'base',
+          'basefont',
+          'bdo',
+          'big',
+          'blockquote',
+          'body',
+          'br',
+          'button',
+          'caption',
+          'center',
+          'cite',
+          'code',
+          'col',
+          'colgroup',
+          'dd',
+          'del',
+          'dfn',
+          'dir',
+          'div',
+          'dl',
+          'dt',
+          'em',
+          'fieldset',
+          'font',
+          'form',
+          'frame',
+          'frameset',
+          'h1',
+          'h2',
+          'h3',
+          'h4',
+          'h5',
+          'h6',
+          'head',
+          'hr',
+          'html',
+          'i',
+          'iframe',
+          'img',
+          'input',
+          'ins',
+          'isindex',
+          'kbd',
+          'label',
+          'legend',
+          'li',
+          'link',
+          'map',
+          'menu',
+          'meta',
+          'noframes',
+          'noscript',
+          'object',
+          'ol',
+          'optgroup',
+          'option',
+          'p',
+          'param',
+          'pre',
+          'q',
+          's',
+          'samp',
+          'select',
+          'small',
+          'span',
+          'strike',
+          'strong',
+          'style',
+          'sub',
+          'sup',
+          'table',
+          'tbody',
+          'td',
+          'textarea',
+          'tfoot',
+          'th',
+          'thead',
+          'title',
+          'tr',
+          'tt',
+          'u',
+          'ul',
+          'var',
+          'xmp'
+  ]
+end

data/lib/despamilator/filter/naughty_q.rb

@@ -0,0 +1,17 @@
+def name
+  'Naughty Q'
+end
+
+def description
+  'Detects possible misuse of the letter Q (English language)'
+end
+
+def parse
+  matches = self.text.downcase.scan(/q./)
+
+  return unless matches
+
+  matches.each do |match|
+    self.append_score = 0.2 if match != 'qu' and match != 'qa' and match !~ /q\s/
+  end
+end

data/lib/despamilator/filter/numbers_and_words.rb

@@ -0,0 +1,33 @@
+def parse
+  string = self.text.downcase
+
+  # strip out "good numbers"
+  string.gsub!(/h[1-6]/, '')
+  string.gsub!(/(^|\b)\d+($|\b)/, '')
+  string.gsub!(/(^|\b)\d+(,|\.)\d+($|\b)/, '')
+  string.gsub!(/(^|\b)\d+(st|nd|rd|th)($|\b)/, '')
+
+  [
+          /\w\d+/,
+          /\d+\w/,
+          /\d+($|\b)/
+  ].each do |regexp|
+    matches = string.scan(regexp)
+
+    next if matches.empty?
+
+    matches.each do |to_remove|
+      to_remove = to_remove.to_s
+      string.sub!(to_remove, '') unless to_remove.empty?
+      self.append_score = 0.1
+    end
+  end
+end
+
+def name
+  'Numbers next to words'
+end
+
+def description
+  'Detects unusual number/word combinations'
+end

data/lib/despamilator/filter/script_tag.rb

@@ -0,0 +1,13 @@
+def parse
+  if self.text.downcase.match(/<\/?script(>|\s+|\n|\r)/)
+    self.append_score = 1
+  end
+end
+
+def name
+  'Detects script tags in text'
+end
+
+def description
+  'Searches for variations for the HTML script tag'
+end

data/lib/despamilator/filter.rb

@@ -0,0 +1,52 @@
+class Despamilator
+  class Filter
+    attr_accessor :matches, :score
+
+    def initialize text
+      @@loaded ||= {}
+      @filters ||= []
+      @matches ||= []
+      @score ||= 0
+      load_filters text
+      run_filters
+    end
+
+    private
+
+    def load_filters text
+      Dir.glob(File.dirname(__FILE__) + "/filter/*.rb").each do |filter_file|
+        filter_name = classify_filename filter_file
+        filter = @@loaded[filter_name]
+
+        unless filter
+          filter_code = File.open(filter_file, File::RDWR).read
+          filter = Class.new
+          filter.class_eval(
+                  "require 'despamilator/filter_base'\nclass #{filter_name} < Despamilator::FilterBase\n#{filter_code}\nend"
+          )
+        end
+
+        @filters.push(filter.const_get(filter_name).new(text.to_s.dup, File.basename(filter_file)))
+      end
+    end
+
+    def run_filters
+      @filters.each do |filter|
+        filter.parse
+
+        if filter.matched?
+          @matches.push(filter)
+          @score += filter.score
+        end
+      end
+    end
+
+    def classify_filename filename
+      classname = ''
+      File.basename(filename).gsub(/\.rb$/, '').split('_').each do |filename_part|
+        classname += filename_part.capitalize
+      end
+      classname || filename.capitalize
+    end
+  end
+end