despamilator 2.0.1 → 2.1

data/History.txt

@@ -63,3 +63,10 @@
 
 * Rejigged url and html tag scores.
 
+=== 2.0.1
+
+* Minor bug fix in shouting filter.
+
+=== IN PROGRESS
+
+* Removed the "matched_by" method. Replaced by "matches".

data/lib/despamilator.rb

@@ -1,6 +1,11 @@
 $:.unshift(File.dirname(__FILE__)) unless $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
-require 'despamilator/filter'
+Dir.glob(File.join(File.dirname(__FILE__), 'despamilator', 'filter', '*.rb')).each do |filter_file|
+  require filter_file
+end
+
+require 'despamilator/subject'
+require 'ostruct'
 
 #== SYNOPSIS:
 #
@@ -14,25 +19,39 @@ require 'despamilator/filter'
 #  dspam.matched_by #=> array of matching filters
 
 class Despamilator
-  VERSION = "2.0.1"
 
   # Constructor. Takes the text you which to parse and score.
 
   def initialize text
-    @filters = Despamilator::Filter.new text
+    @subject = Despamilator::Subject.new text
+    run_filters @subject
   end
 
   # Returns the total score as a Float.
 
   def score
-    @filters.score
+    @subject.score
+  end
+
+  def matched_by
+    warn 'Despamilator.matched_by is deprecated, please use Despamilator.matches by 2011-12-31.'
+
+    matches.map do |match|
+      filter = match[:filter]
+
+      OpenStruct.new(
+          :name => filter.name,
+          :description => filter.description,
+          :score => match[:score]
+      )
+    end
   end
 
-  # Returns an array of filters that have matched and contributed to the score.
+  # Returns an array of scores and filters that have matched and contributed to the score.
   # Each element is a a child of the Despamilator::FilterBase class.
 
-  def matched_by
-    @filters.matches
+  def matches
+    @subject.matches
   end
 
   # Generic Test for Unsolicited Bulk Submissions. Similar to SpamAssassin's GTUBE.
@@ -41,4 +60,16 @@ class Despamilator
   def self.gtubs_test_string
     '89913b8a065b7092721fe995877e097681683af9d3ab767146d5d6fd050fc0bda7ab99f4232d94a1'
   end
+
+  private
+
+  def run_filters subject
+    filter_namespace = Object.const_get('DespamilatorFilter')
+
+    filter_namespace.constants.each do |filter_class|
+      filter = filter_namespace.const_get(filter_class).new
+      filter.parse(subject)
+    end
+  end
+
 end

data/lib/despamilator/filter.rb

@@ -1,36 +1,52 @@
 class Despamilator
-  class Filter
-    attr_accessor :matches, :score
 
-    def initialize text
-      @matches ||= []
-      @score ||= 0
-      run_filters text
-    end
+#This class is the base class of all the despamilator filters.
+#
+#== EXAMPLE:
+#
+#This example is to detect the letter "a". Put the code in
+#lib/despamilator/filter/detect_letter_a.rb:
+#
+#  require 'despamilator/filter_base'
+#
+#  module DespamilatorFilter
+#
+#    class DetectLetterA < Despamilator::FilterBase
+#
+#      def name
+#        'Detecting the letter A'
+#      end
+#
+#      def description
+#        'Detects the letter "a" in a string for no reason other than a demo'
+#      end
+#
+#      def parse text
+#        if text.downcase.scan(/a/)
+#        # add 0.1 to the score of the text
+#        self.append_score = 0.1
+#      end
+#    end
+#  end
 
-    private
+  class Filter
 
-    def run_filters text
-      filter_namespace = Object.const_get('DespamilatorFilter')
+    # The nice description of the filter. Usually no more than a sentence.
 
-      filter_namespace.constants.each do |filter_class|
-        execute_filter(filter_namespace.const_get(filter_class).new, text)
-      end
+    def description
+      raise "No description defined for #{self.class}"
     end
 
-    private
+    # This method parses some text. The score is assigned to the same instance.
 
-    def execute_filter filter, text
-      filter.parse text.dup
-
-      if filter.matched?
-        @matches.push(filter)
-        @score += filter.score
-      end
+    def parse text
+      raise "No parser defined for #{self.class}"
     end
 
-    Dir.glob(File.join(File.dirname(__FILE__), 'filter', '*.rb')).each do |filter_file|
-      require filter_file
+    # The one or two word name for the filter.
+
+    def name
+      raise "No name defined for #{self.class}"
     end
 
   end

data/lib/despamilator/filter/gtubs_test_filter.rb

@@ -1,8 +1,8 @@
-require 'despamilator/filter_base'
+require 'despamilator/filter'
 
 module DespamilatorFilter
 
-  class GtubsTestFilter < Despamilator::FilterBase
+  class GtubsTestFilter < Despamilator::Filter
 
     def name
       'GTubs Test Filter'
@@ -12,8 +12,8 @@ module DespamilatorFilter
       'Detects the special test string (Despamilator.gtubs_test_string) and assigns a big score.'
     end
 
-    def parse text
-      self.append_score = 100 if text == Despamilator.gtubs_test_string
+    def parse subject
+      subject.register_match!({:score => 100, :filter => self}) if subject.text == Despamilator.gtubs_test_string
     end
 
   end

data/lib/despamilator/filter/html_tags.rb

@@ -1,15 +1,17 @@
-require 'despamilator/filter_base'
-
 module DespamilatorFilter
 
-  class HtmlTags < Despamilator::FilterBase
+  class HtmlTags < Despamilator::Filter
 
-    def parse text
-      text.downcase!
+    def parse subject
+      text = subject.text.downcase
 
       html_tags.each do |tag|
-        if text.match(/<\s*#{tag}\W/) || text.match(/<\n*#{tag}\W/) || text.match(/\W#{tag}\s*\//) || text.match(/\W#{tag}\n*\//)
-          self.append_score = 0.6
+        opening_elements = text.count(/<\s*#{tag}\W/)
+        closing_elements = text.count(/\W#{tag}\s*\/>/)
+
+        if opening_elements > 0 or closing_elements > 0
+          safest_element_count = opening_elements > closing_elements ? opening_elements : closing_elements
+          subject.register_match!({:score => 0.6 * safest_element_count, :filter => self})
         end
       end
     end

data/lib/despamilator/filter/ip_address_url.rb

@@ -1,8 +1,8 @@
-require 'despamilator/filter_base'
+require 'despamilator/filter'
 
 module DespamilatorFilter
 
-  class IPAddressURL < Despamilator::FilterBase
+  class IPAddressURL < Despamilator::Filter
 
     def name
       'IP Address URL'
@@ -12,8 +12,10 @@ module DespamilatorFilter
       'Detects IP address URLs'
     end
 
-    def parse text
-      self.append_score = 0.5 if text.downcase.scan(/http:\/\/\d+\.\d+\.\d+\.\d+/).length > 0
+    def parse subject
+      subject.register_match!({
+          :score => 0.5, :filter => self
+      }) if subject.text.downcase.count(/http:\/\/\d+\.\d+\.\d+\.\d+/) > 0
     end
 
   end

data/lib/despamilator/filter/long_words.rb

@@ -1,8 +1,8 @@
-require 'despamilator/filter_base'
+require 'despamilator/filter'
 
 module DespamilatorFilter
 
-  class LongWords < Despamilator::FilterBase
+  class LongWords < Despamilator::Filter
 
     def name
       'Long Words'
@@ -12,9 +12,11 @@ module DespamilatorFilter
       'Detects long and unbroken strings'
     end
 
-    def parse text
-      text.split(/\W+/).each do |word|
-        self.append_score = 0.1 if word.length > 20
+    def parse subject
+      subject.text.without_uris.words.each do |word|
+        subject.register_match!({
+          :score => 0.1, :filter => self
+        }) if word.length > 20
       end
     end
 

data/lib/despamilator/filter/mixed_case.rb

@@ -0,0 +1,21 @@
+module DespamilatorFilter
+
+  class MixedCase < Despamilator::Filter
+    def name
+      'Mixed Case String'
+    end
+
+    def description
+      'Detects mixed case strings.'
+    end
+
+    def parse subject
+      text = subject.text.without_uris
+      count = text.remove_and_count!(/[a-z][A-Z]/)
+      count += text.remove_and_count!(/[a-z][A-Z][a-z]/)
+      subject.register_match!({:score => 0.1 * count, :filter => self}) if count > 0
+    end
+
+  end
+
+end

data/lib/despamilator/filter/naughty_words.rb

@@ -1,8 +1,8 @@
-require 'despamilator/filter_base'
+require 'despamilator/filter'
 
 module DespamilatorFilter
 
-  class NaughtyWords < Despamilator::FilterBase
+  class NaughtyWords < Despamilator::Filter
 
     def name
       'Naughty Words'
@@ -12,11 +12,11 @@ module DespamilatorFilter
       'Detects cheeky words'
     end
 
-    def parse text
-      text.downcase!
+    def parse subject
+      text = subject.text.downcase
 
       naughty_words.each do |word|
-        self.append_score = 0.1 if text =~ /\b#{word}s?\b/
+        subject.register_match!({:score => 0.1, :filter => self}) if text =~ /\b#{word}s?\b/
       end
     end
 

data/lib/despamilator/filter/numbers_and_words.rb

@@ -1,17 +1,11 @@
-require 'despamilator/filter_base'
+require 'despamilator/filter'
 
 module DespamilatorFilter
 
-  class NumbersAndWords < Despamilator::FilterBase
+  class NumbersAndWords < Despamilator::Filter
 
-    def parse text
-      text.downcase!
-
-      # strip out "good numbers"
-      text.gsub!(/h[1-6]/, '')
-      text.gsub!(/(^|\b)\d+($|\b)/, '')
-      text.gsub!(/(^|\b)\d+(,|\.)\d+($|\b)/, '')
-      text.gsub!(/(^|\b)\d+(st|nd|rd|th)($|\b)/, '')
+    def parse subject
+      text = tidy_text(subject)
 
       [
               /\w\d+/,
@@ -25,7 +19,7 @@ module DespamilatorFilter
         matches.each do |to_remove|
           to_remove = to_remove.to_s
           text.sub!(to_remove, '') unless to_remove.empty?
-          self.append_score = 0.1
+          subject.register_match!({:score => 0.1, :filter => self})
         end
       end
     end
@@ -38,6 +32,20 @@ module DespamilatorFilter
       'Detects unusual number/word combinations'
     end
 
+    private
+
+    def tidy_text subject
+      text = subject.text.without_uris
+      text.downcase!
+
+      # strip out "good numbers"
+      text.gsub!(/h[1-6]/, '')
+      text.gsub!(/(^|\b)\d+($|\b)/, '')
+      text.gsub!(/(^|\b)\d+(,|\.)\d+($|\b)/, '')
+      text.gsub!(/(^|\b)\d+(st|nd|rd|th)($|\b)/, '')
+
+      text
+    end
   end
 
 end

data/lib/despamilator/filter/obfuscated_urls.rb

@@ -0,0 +1,41 @@
+module DespamilatorFilter
+
+  class ObfuscatedURLs < Despamilator::Filter
+    def name
+      'Obfuscated URLs'
+    end
+
+    def description
+      'Finds lame attempts at obfuscating urls.'
+    end
+
+    def parse subject
+      text = subject.text.without_uris.downcase
+      count = find_space_separated_parts text
+      count += find_space_separated_characters text
+
+      # weird maths below is due to some issue with ruby 1.9.2 multiplying floats by 3 (?!)
+      subject.register_match!({score: (4.0 * count) / 10, filter: self}) if count > 0
+    end
+
+    private
+
+    def find_space_separated_parts text
+      text.count(/www\s+\w+\s+com/)
+    end
+
+    def find_space_separated_characters text
+      count = 0
+
+      text.split(/[a-z][a-z]/).each do |candidate|
+        candidate.strip!
+        candidate.gsub!(/\s+/, '')
+        count += 1 if candidate =~ /\w{5,}\.\w{2,3}/
+      end
+
+      count
+    end
+
+  end
+
+end

data/lib/despamilator/filter/prices.rb

@@ -0,0 +1,19 @@
+module DespamilatorFilter
+
+  class Prices < Despamilator::Filter
+    def name
+      'Prices'
+    end
+
+    def description
+      'Detects prices in text.'
+    end
+
+    def parse subject
+      price_count = subject.text.count(/\$\s*\d+/)
+      subject.register_match!({:score => 0.075 * price_count, :filter => self}) if price_count > 0
+    end
+
+  end
+
+end

data/lib/despamilator/filter/script_tag.rb

@@ -1,11 +1,11 @@
-require 'despamilator/filter_base'
+require 'despamilator/filter'
 
 module DespamilatorFilter
 
-  class ScriptTag < Despamilator::FilterBase
+  class ScriptTag < Despamilator::Filter
 
-    def parse text
-      self.append_score = 1 if text.downcase.match(/<\/?script(>|\s+|\n|\r)/)
+    def parse subject
+      subject.register_match!({:score => 1, :filter => self}) if subject.text.downcase.match(/<\/?script(>|\s+|\n|\r)/)
     end
 
     def name

data/lib/despamilator/filter/shouting.rb

@@ -1,8 +1,8 @@
-require 'despamilator/filter_base'
+require 'despamilator/filter'
 
 module DespamilatorFilter
 
-  class Shouting < Despamilator::FilterBase
+  class Shouting < Despamilator::Filter
 
     def name
       'Shouting'
@@ -12,17 +12,20 @@ module DespamilatorFilter
       'Detects and scores shouting (all caps)'
     end
 
-    def parse text
+    def parse subject
       # strip HTML
-      text.gsub!(/<\/?[^>]*>/, "")
+      text = subject.text.gsub(/<\/?[^>]*>/, "")
 
       return if text.length < 20
 
       uppercased = text.scan(/[A-Z][A-Z]+/).join.length
-      lowercased = text.scan(/[a-z]/).length
+      lowercased = text.count(/[a-z]/)
 
       if uppercased > 0
-        self.append_score = (uppercased.to_f / (uppercased + lowercased)) * 0.5
+        subject.register_match!({
+            :score => (uppercased.to_f / (uppercased + lowercased)) * 0.5,
+            :filter => self
+        })
       end
     end