despamilator 2.0.1 → 2.1

data/lib/despamilator/filter/spammy_tlds.rb

@@ -0,0 +1,22 @@
+require 'despamilator/filter'
+
+module DespamilatorFilter
+
+  class SpammyTLDs < Despamilator::Filter
+
+    def name
+      'Spammy TLDs'
+    end
+
+    def description
+      'Detects TLDs that are more commonly associated with spam.'
+    end
+
+    def parse subject
+      matches = subject.text.count(/\w{5,}\.(info|biz)\b/)
+      subject.register_match!({:score => 0.05 * matches, :filter => self}) if matches > 0
+    end
+
+  end
+
+end

data/lib/despamilator/filter/square_brackets.rb

@@ -1,8 +1,8 @@
-require 'despamilator/filter_base'
+require 'despamilator/filter'
 
 module DespamilatorFilter
 
-  class SquareBrackets < Despamilator::FilterBase
+  class SquareBrackets < Despamilator::Filter
 
     def name
       'Square Brackets'
@@ -12,9 +12,9 @@ module DespamilatorFilter
       'Detects each square bracket in a string'
     end
 
-    def parse text
-      text.downcase.scan(/(\[|\])/).each do |match|
-        self.append_score = 0.05
+    def parse subject
+      subject.text.downcase.scan(/(\[|\])/).each do |match|
+        subject.register_match!({:score => 0.05, :filter => self})
       end
     end
 

data/lib/despamilator/filter/trailing_number.rb

@@ -1,8 +1,8 @@
-require 'despamilator/filter_base'
+require 'despamilator/filter'
 
 module DespamilatorFilter
 
-  class TrailingNumber < Despamilator::FilterBase
+  class TrailingNumber < Despamilator::Filter
 
     def name
       'Trailing Number'
@@ -12,8 +12,8 @@ module DespamilatorFilter
       'Detects a trailing cache busting number'
     end
 
-    def parse text
-      self.append_score = 0.1 if text =~ /\b\d+\s*$/
+    def parse subject
+      subject.register_match!({:score => 0.1, :filter => self}) if subject.text.without_uris =~ /\b\d+\s*$/
     end
 
   end

data/lib/despamilator/filter/unusual_characters.rb

@@ -1,8 +1,8 @@
-require 'despamilator/filter_base'
+require 'despamilator/filter'
 
 module DespamilatorFilter
 
-  class UnusualCharacters < Despamilator::FilterBase
+  class UnusualCharacters < Despamilator::Filter
 
     def name
       'Unusual Characters'
@@ -12,10 +12,10 @@ module DespamilatorFilter
       'Detects and scores each occurrence of an unusual 2 or 3 character combination'
     end
 
-    def parse text
+    def parse subject
       initialize_combos
-      tokenize(text).each do |token|
-        self.append_score = 0.05 if @@combos[token.to_sym]
+      tokenize(subject.text.without_uris).each do |token|
+        subject.register_match!({:score => 0.05, :filter => self}) if @@combos[token.to_sym]
       end
     end
 

data/lib/despamilator/filter/urls.rb

@@ -1,8 +1,8 @@
-require 'despamilator/filter_base'
+require 'despamilator/filter'
 
 module DespamilatorFilter
 
-  class URLs < Despamilator::FilterBase
+  class URLs < Despamilator::Filter
 
     def name
       'URLs'
@@ -12,13 +12,11 @@ module DespamilatorFilter
       'Detects each url in a string'
     end
 
-    def parse text
-      text.downcase!
-
-      text.gsub!(/http:\/\/\d+\.\d+\.\d+\.\d+/, '')
-
-      1.upto(text.scan(/http:\/\//).length) do
-        self.append_score = 0.4
+    def parse subject
+      text = subject.text.downcase.gsub(/http:\/\/\d+\.\d+\.\d+\.\d+/, '')
+      matches = text.count(/https?:\/\//)
+      1.upto(matches > 2 ? 2 : matches) do
+        subject.register_match!({:score => 0.4, :filter => self})
       end
     end
 

data/lib/despamilator/filter/very_long_domain_name.rb

@@ -0,0 +1,27 @@
+require 'despamilator/filter'
+require 'domainatrix'
+
+module DespamilatorFilter
+
+  class VeryLongDomainName < Despamilator::Filter
+
+    def name
+      'Very Long Domain Name'
+    end
+
+    def description
+      'Detects unusually long domain names.'
+    end
+
+    def parse subject
+      subject.text.scan(URI.regexp).each do |url_parts|
+        url_parts.compact!
+        next if !url_parts[1] or url_parts[1] !~ /(\w|-){5,}\.\w{2,5}/
+        url = Domainatrix.parse('http://' + url_parts[1])
+        subject.register_match!({:score => 0.4, :filter => self}) if url.domain.length > 20
+      end
+    end
+
+  end
+
+end

data/lib/despamilator/filter/weird_punctuation.rb

@@ -0,0 +1,44 @@
+require 'despamilator/filter'
+
+module DespamilatorFilter
+
+  class WeirdPunctuation < Despamilator::Filter
+
+    def name
+      'Weird Punctuation'
+    end
+
+    def description
+      'Detects unusual use of punctuation.'
+    end
+
+    def parse subject
+      text = subject.text.without_uris.downcase
+
+      text.gsub!(/\w&\w/, 'xx')
+      text.gsub!(/[a-z](!|\?)(\s|$)/, 'x')
+      text.gsub!(/(?:#{punctuation}){20,}/, '')
+      matches = text.remove_and_count!(/(?:\W|\s|^)(#{punctuation})/)
+      matches += text.remove_and_count!(/\w,\w/)
+      matches += text.remove_and_count!(/\w\w\.\w/)
+      matches += text.remove_and_count!(/\w\.\w\w/)
+      matches += text.remove_and_count!(/(#{punctuation})(#{punctuation})/)
+      matches += text.remove_and_count!(/(#{punctuation})$/)
+      matches += text.remove_and_count!(/(?:\W|\s|^)\d+(#{punctuation})/)
+
+      subject.register_match!({:score => 0.03 * matches, :filter => self}) if matches > 0
+    end
+
+    private
+
+    def punctuation
+      @punctuation ||= %w{~ ` ! @ # $ % ^ & * _ - + = , / ? | \\ : ;}.map do |punctuation_character|
+        Regexp.escape(punctuation_character)
+      end.join('|')
+
+      @punctuation
+    end
+
+  end
+
+end

data/lib/despamilator/subject.rb

@@ -0,0 +1,30 @@
+require 'despamilator/subject/text'
+
+class Despamilator
+  class Subject
+    attr_reader :score, :text
+
+    def initialize text
+      @score = 0.0
+      @matches = {}
+      @text = Despamilator::Subject::Text.new(text)
+    end
+
+    def register_match! details
+      @score += details[:score] || raise('A score must be supplied')
+      filter = details[:filter] || raise('A filter must be supplied')
+
+      @matches[filter] ||= 0.0
+      @matches[filter] += details[:score]
+    end
+
+    def matches
+      @matches.map do |filter, score|
+        {:filter => filter, :score => score}
+      end.sort do |a, b|
+        b[:score] <=> a[:score]
+      end
+    end
+
+  end
+end

data/lib/despamilator/subject/text.rb

@@ -0,0 +1,32 @@
+require 'uri'
+
+class Despamilator
+  class Subject
+    class Text < String
+
+      def initialize text
+        super text
+        freeze
+      end
+
+      def without_uris
+        gsub(/\b(?:https?|mailto|ftp):.+?(\s|$)/i, '')
+      end
+
+      def words
+        split(/\W+/)
+      end
+
+      def count pattern
+        scan(pattern).flatten.compact.length
+      end
+
+      def remove_and_count! pattern
+        count = count(pattern)
+        gsub!(pattern, '')
+        count
+      end
+
+    end
+  end
+end

data/lib/despamilator/version.rb

@@ -0,0 +1,3 @@
+class Despamilator
+  VERSION = 2.1
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: despamilator
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: '2.1'
   prerelease: 
 platform: ruby
 authors:
@@ -9,102 +9,58 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-08-11 00:00:00.000000000 -04:00
-default_executable: 
+date: 2011-09-11 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rubyforge
-  requirement: &2730850 !ruby/object:Gem::Requirement
+  name: domainatrix
+  requirement: &70312466224620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 2.0.4
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
-  version_requirements: *2730850
-- !ruby/object:Gem::Dependency
-  name: hoe
-  requirement: &2730610 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 2.7.0
-  type: :development
-  prerelease: false
-  version_requirements: *2730610
+  version_requirements: *70312466224620
 description: ! 'Despamilator is a plugin based spam detector designed for use on your
-  web forms borne out of two annoyances:
-
-  Spam being submitted in my web forms and CAPTCHAS being intrusive. Despamilator
-  will apply
-
-  some commonly used heuristics from the world of anti-spam to help you decide whether
-  your users are human or machine.'
+  web forms borne out of two annoyances: Spam being submitted in my web forms and
+  CAPTCHAS being intrusive. Despamilator will apply some commonly used heuristics
+  from the world of anti-spam to help you decide whether your users are human or machine.'
 email:
 - moowahaha@hotmail.com
 executables: []
 extensions: []
-extra_rdoc_files:
-- History.txt
-- Manifest.txt
-- PostInstall.txt
-- conf/unusual_characters.txt
+extra_rdoc_files: []
 files:
-- .rspec
-- .rvmrc
-- Gemfile
-- Gemfile.lock
-- History.txt
-- Manifest.txt
-- PostInstall.txt
-- README.rdoc
-- Rakefile
-- conf/unusual_characters.txt
-- despamilator.gemspec
-- lib/despamilator.rb
-- lib/despamilator/filter.rb
 - lib/despamilator/filter/gtubs_test_filter.rb
 - lib/despamilator/filter/html_tags.rb
 - lib/despamilator/filter/ip_address_url.rb
 - lib/despamilator/filter/long_words.rb
+- lib/despamilator/filter/mixed_case.rb
 - lib/despamilator/filter/naughty_words.rb
 - lib/despamilator/filter/numbers_and_words.rb
+- lib/despamilator/filter/obfuscated_urls.rb
+- lib/despamilator/filter/prices.rb
 - lib/despamilator/filter/script_tag.rb
 - lib/despamilator/filter/shouting.rb
+- lib/despamilator/filter/spammy_tlds.rb
 - lib/despamilator/filter/square_brackets.rb
 - lib/despamilator/filter/trailing_number.rb
 - lib/despamilator/filter/unusual_characters.rb
 - lib/despamilator/filter/urls.rb
-- lib/despamilator/filter_base.rb
-- scripts/despamilator_score.rb
-- scripts/from_file.rb
-- spec/despamilator_spec.rb
-- spec/filter_base_spec.rb
-- spec/filters/gtubs_test_filter_spec.rb
-- spec/filters/html_tags_spec.rb
-- spec/filters/ip_address_url_spec.rb
-- spec/filters/long_words_spec.rb
-- spec/filters/naughty_words_spec.rb
-- spec/filters/numbers_and_words_spec.rb
-- spec/filters/script_tag_spec.rb
-- spec/filters/shouting_spec.rb
-- spec/filters/square_brackets_spec.rb
-- spec/filters/trailing_number_spec.rb
-- spec/filters/unusual_characters_spec.rb
-- spec/filters/urls_spec.rb
-- spec/helpers/corpus_helper.rb
-- spec/helpers/filter_helper.rb
-- spec/helpers/spec_helper.rb
-- tasks/test.rake
-has_rdoc: true
+- lib/despamilator/filter/very_long_domain_name.rb
+- lib/despamilator/filter/weird_punctuation.rb
+- lib/despamilator/filter.rb
+- lib/despamilator/subject/text.rb
+- lib/despamilator/subject.rb
+- lib/despamilator/version.rb
+- lib/despamilator.rb
+- README.rdoc
+- History.txt
 homepage: http://github.com/moowahaha/despamilator
 licenses: []
-post_install_message: PostInstall.txt
-rdoc_options:
-- --main
-- README.rdoc
+post_install_message: 
+rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
@@ -118,13 +74,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.6
 requirements: []
 rubyforge_project: despamilator
-rubygems_version: 1.5.2
+rubygems_version: 1.8.6
 signing_key: 
 specification_version: 3
-summary: ! 'Despamilator is a plugin based spam detector designed for use on your
-  web forms borne out of two annoyances: Spam being submitted in my web forms and
-  CAPTCHAS being intrusive'
+summary: Stop web form Spam!
 test_files: []

data/.rspec

@@ -1,2 +0,0 @@
---color
---require=./spec/helpers/spec_helper.rb

data/.rvmrc

@@ -1 +0,0 @@
-rvm --create use 1.9.2@despamilator

data/Gemfile

@@ -1,12 +0,0 @@
-# A sample Gemfile
-source "http://rubygems.org"
-
-gem 'hoe', '>= 2.7.0'
-gem 'newgem', '>= 1.5.3'
-gem 'rdoc', '>= 3.2'
-
-group :test do
-  gem 'rspec', '>= 2.0.1'
-  gem 'simplecov', '>= 0.3.7'
-  gem 'one_hundred_percent_coverage'
-end