highscore 0.5.0 → 0.5.2

data/.travis.yml

@@ -6,5 +6,7 @@ rvm:
   - jruby-18mode # JRuby in 1.8 mode
   - jruby-19mode # JRuby in 1.9 mode
   - rbx-18mode
-  # - rbx-19mode # currently in active development, may or may not work for your project
-before_install: gem install bones
+  - rbx-19mode
+
+# fast-stemmer is a C extension, so it won't work with JRuby ...
+before_install: gem install bones; (gem install fast-stemmer &); sleep 10

data/History.txt

@@ -1,3 +1,9 @@
+== 0.5.2 / 2012-02-25
+
+* added (optional) support for word stemming using the fast-stemmer gem
+* setting to enable/disable case-sensitivy (thanks to carlosramireziii)
+* use other regex patterns to cut the text in keywords (thanks to carlosramireziii)
+
 == 0.5.0 / 2012-02-20
 
 * now supports a whitelist approach

data/README.md

@@ -1,7 +1,9 @@
 highscore
 ===========
 
-Find and rank keywords in long texts.
+Easily find and rank keywords in long texts.
+
+[![Build Status](https://secure.travis-ci.org/domnikl/highscore.png?branch=develop)](http://travis-ci.org/domnikl/highscore)
 
 Features
 --------
@@ -11,6 +13,7 @@ Features
 * directly get keywords from String objects
 * blacklist words via a plain text file, String or an Array of words
 * optionally, configure a whitelist and only words from that list will get ranked
+* use word stemming (requires the fast-stemmer gem, doesn't work on JRuby platforms!)
 * merge together Keywords from multiple sources
 * contains a CLI tool that operates on STDIN/OUT and is configurable via parameters
 
@@ -24,8 +27,11 @@ text.configure do
   set :upper_case, 3
   set :long_words, 2
   set :long_words_threshold, 15
-  set :vowels, 1 # => default = 0 = not considered
-  set :consonants, 5 # => default = 0 = not considered
+  set :vowels, 1                     # => default: 0 = not considered
+  set :consonants, 5                 # => default: 0 = not considered
+  set :ignore_case, true             # => default: false
+  set :word_pattern, /[\w]+[^\s0-9]/ # => default: /\w+/
+  set :stemming, true                # => default: false
 end
 
 # get only the top 50 keywords
@@ -94,6 +100,10 @@ Install
 
 * `[sudo] gem install highscore`
 
+To use word stemming, you need to have the fast-stemmer gem installed:
+
+* `[sudo] gem install fast-stemmer`
+
 Author
 ------
 

data/Rakefile

@@ -1,3 +1,4 @@
+require 'rubygems'
 
 begin
   require 'bones'
@@ -15,4 +16,3 @@ Bones {
   url      'http://thewebdev.de'
   ignore_file '.gitignore'
 }
-

data/bin/highscore

@@ -31,14 +31,16 @@ optparse = OptionParser.new do |opts|
     options[:emphasis][:multiplier] = multiplier.to_f
   end
 
-  # don't print rank weight
   opts.on('--no-ignore-short', 'don\'t ignore short words (<= 2 chars)') do
     options[:emphasis][:ignore_short_words] = false
   end
 
-  # don't print rank weight
-  opts.on('-s', '--short', 'don\'t print rank weight') do
-    options[:short] = true
+  opts.on('--ignore-case', 'case insensitive') do
+    options[:emphasis][:ignore_case] = true
+  end
+
+  opts.on('--stemming', 'use word stemming (won\'t work on JRuby platforms)') do
+    options[:emphasis][:stemming] = true
   end
 
   # upper case
@@ -69,6 +71,11 @@ optparse = OptionParser.new do |opts|
   opts.on('-n', '--top N', 'show only the top N keywords') do |u|
     options[:top] = u.to_i
   end
+
+  # don't print rank weight
+  opts.on('-s', '--short', 'don\'t print rank weight') do
+    options[:short] = true
+  end
 end
 
 optparse.parse!

data/lib/highscore/content.rb

@@ -1,6 +1,9 @@
-$:.unshift(File.join(File.dirname(__FILE__)))
+$:.unshift(File.dirname(__FILE__))
 require 'keywords'
 
+# external gems
+require 'rubygems'
+
 module Highscore
   class Content
     attr_reader :content
@@ -26,7 +29,10 @@ module Highscore
         :long_words_threshold => 15,
         :vowels => 0,
         :consonants => 0,
-        :ignore_short_words => true
+        :ignore_short_words => true,
+        :ignore_case => false,
+        :word_pattern => /\w+/,
+        :stemming => false
       }
     end
 
@@ -49,10 +55,13 @@ module Highscore
     #
     # @return Highscore::Keywords
     def keywords
+      @emphasis[:stemming] = use_stemming?
+
       keywords = Keywords.new
 
-      Keywords.find_keywords(@content, wordlist).each do |text|
+      Keywords.find_keywords(processed_content, wordlist, word_pattern).each do |text|
         text = text.to_s
+        text = text.stem if @emphasis[:stemming]
 
         if not (text.match(/^[\d]+(\.[\d]+){0,1}$/) or text.length <= 2)
           keywords << Highscore::Keyword.new(text, weight(text))
@@ -77,6 +86,16 @@ module Highscore
 
     private
 
+    # processes the text content applying any necessary transformations
+    #
+    # @return String
+    def processed_content
+      "".tap do |result|
+        result.replace(@content) # initialize the result to be @content
+        result.replace(result.downcase) if @emphasis[:ignore_case]
+      end
+    end
+
     # allow short words to be rated
     #
     # @return TrueClass FalseClass
@@ -84,6 +103,13 @@ module Highscore
       not @emphasis[:ignore_short_words]
     end
 
+    # regex used to split text
+    #
+    # @return Regex
+    def word_pattern
+      @emphasis[:word_pattern]
+    end
+
     # weight a single text keyword
     #
     # @param text String
@@ -121,5 +147,22 @@ module Highscore
       percent = text.consonants.length / text.length.to_f
       percent * @emphasis[:consonants]
     end
+
+    private
+
+    # using stemming is only possible, if fast-stemmer is installed
+    # doesn't work for JRuby
+    def use_stemming?
+      if @emphasis[:stemming]
+        begin
+          require 'fast_stemmer'
+          true
+        rescue LoadError
+          false
+        end
+      else
+        false
+      end
+    end
   end
 end

data/lib/highscore/keywords.rb

@@ -12,10 +12,10 @@ module Highscore
     #
     # @param content String
     # @param wordlist Highscore::Wordlist
+    # @param pattern Regex
     # @return Highscore::Keywords
-    def self.find_keywords content, wordlist
-      keywords = content.to_s.scan(/\w+/)
-
+    def self.find_keywords content, wordlist, pattern=/\w+/
+      keywords = content.to_s.scan(pattern)
       keywords.delete_if do |key, value|
         if wordlist.kind_of? Highscore::Blacklist
           wordlist.include?(key.downcase)

data/test/highscore/test_content.rb

@@ -1,6 +1,7 @@
 $:.unshift(File.join(File.dirname(__FILE__), %w{.. .. lib highscore}))
 require "content"
 require "test/unit"
+require 'rubygems'
 
 class TestContent < Test::Unit::TestCase
   def setup
@@ -50,4 +51,38 @@ class TestContent < Test::Unit::TestCase
 
     assert_equal 4, keywords.length
   end
+
+  def test_word_pattern
+    keywords = 'foo Ruby foo Ruby'.keywords do
+      set :word_pattern, /(?=(\b\w+\s\w+\b))/
+    end
+    
+    assert_equal 2, keywords.length
+  end
+
+  def test_ignore_case
+    keywords = 'foo Foo bar Bar'.keywords do
+      set :ignore_case, true
+    end
+
+    assert_equal 2, keywords.length
+  end
+
+  def test_stemming
+    begin
+      require 'fast_stemmer'
+
+      keywords = 'word words boards board woerter wort'.keywords do
+        set :stemming, true
+      end
+
+      assert_equal 4, keywords.length
+
+      keywords.each do |k|
+        assert (%w{board word woerter wort}).include?(k.text)
+      end
+    rescue LoadError
+      # do nothing, just skip this test
+    end
+  end
 end

data/version.txt

@@ -1 +1 @@
-0.5.0
+0.5.2

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: highscore
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-20 00:00:00.000000000 Z
+date: 2012-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bones
-  requirement: &70127917245680 !ruby/object:Gem::Requirement
+  requirement: &70144626889800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,8 +21,8 @@ dependencies:
         version: 3.7.3
   type: :development
   prerelease: false
-  version_requirements: *70127917245680
-description: Find and rank keywords in long texts.
+  version_requirements: *70144626889800
+description: Easily find and rank keywords in long texts.
 email: liebler.dominik@googlemail.com
 executables:
 - highscore
@@ -83,7 +83,7 @@ rubyforge_project: highscore
 rubygems_version: 1.8.16
 signing_key: 
 specification_version: 3
-summary: Find and rank keywords in long texts.
+summary: Easily find and rank keywords in long texts.
 test_files:
 - test/highscore/test_blacklist.rb
 - test/highscore/test_content.rb