sentiment_lib 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.DS_Store

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sentiment_lib.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Nicholas Zaillian
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,90 @@
+# SentimentLib
+
+A simple, extensible sentiment analysis gem.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'sentiment_lib'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sentiment_lib
+
+## Usage
+
+SentimentLib exposes simple sentiment analysis via SentimentLib::Analyzer.
+
+example usage --
+
+    analyzer = SentimentLib::Analyzer.new
+
+    analyzer.analyze("I'm feeling confident and excited this morning!")
+    => 2.0
+
+
+The library ships with two analysis strategies (you will liklely want to add your own domain-specific strategies):
+
+    SentimentLib::Analysis::Strategies::BasicDictStrategy
+
+and
+
+    SentimentLib::Analysis::Strategies::FinancialDictStrategy
+
+The first (BasicDictStrategy) uses a dictionary file (at lib/sentiment_lib/data/analysis/basic_dict/words.txt) mapping words to decimal weights between -1.0 and 1.0.  It's analysis is the average of word weights for the supplied string.
+
+The second (FinancialDictStrategy) uses the Loughran and McDonald Financial Sentiment Dictionary with +1 and -1 weights mapped to the dictionary's positive and negative words, respectively.  It's analysis is the sum of word weights for the supplied string.
+
+The API for adding your own analysis strategies is straightforward:
+
+create a subclass of SentimentLib::Analysis::Strategy
+
+  implement the following public instance methods (all but first optional):
+
+  required: 
+    
+    mappings
+      returns a hash mapping token strings to weights
+      
+      ex:
+      
+      {
+        happy => 1.0,
+        sad => -1.0
+      }
+
+      The supplied strategies both load text files with
+      word-weight mappings and parse them into a hash.
+
+  optional:
+
+    normalize(str)
+      Normalize the string to be analyzed (applied before tokenization - see below).
+      Default is to downcase and strip all alphanumeric characters.
+
+    tokenize(str)
+      Defines custom method for converting the string being analyzed (str) into 
+      tokens (default splits on white space).
+
+    weigh(tokens)
+      Define custom method for weighing the collected tokens of the string
+      (default is to average token weights).
+
+
+You can then pass your strategy to new instances of SentimentLib::Analyzer:
+
+  analyzer = SentimentLib::Analyzer.new(:strategy => MyCustomStrategy.new)
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/sentiment_lib/analysis/sentiment_analyzer.rb

@@ -0,0 +1,11 @@
+module SentimentLib
+  class Analyzer
+    def analyze(str, opts={})
+      @strategy = opts[:strategy] || ::SentimentLib::Analysis::Strategies::FinancialDictStrategy.new
+
+      tokens, sum = @strategy.tokenize(@strategy.normalize(str)), 0
+
+      @strategy.weigh(tokens)
+    end
+  end
+end

data/lib/sentiment_lib/analysis/strategies/basic_dict_strategy.rb

@@ -0,0 +1,57 @@
+require File.expand_path('../../strategy', __FILE__)
+
+module SentimentLib
+  module Analysis
+    module Strategies
+      class BasicDictStrategy < ::SentimentLib::Analysis::Strategy
+        @@mappings = nil
+
+        def initialize
+          self.class._load_mappings if !@@mappings
+        end
+
+        def mappings
+          @@mappings
+        end
+
+        # weight is pure sum (not average)
+        def weigh(tokens, opts={})
+          valid_tokens = tokens.delete_if { |token|
+            mappings[token] == nil
+          }
+
+          return 0 if valid_tokens.length == 0
+
+          sum = 0
+
+          valid_tokens.each { |token| 
+            sum += mappings[token]
+          }
+
+          if opts[:average] == true
+            sum / valid_tokens.length
+          else
+            sum
+          end
+        end         
+
+        private
+
+        def self._load_mappings
+          @@mappings = {}
+
+          data_root = File.expand_path('../../../../sentiment_lib/data', __FILE__)
+          
+          word_file = File.new("#{data_root}/analysis/basic_dict/words.txt", 'r')
+
+          while (line = word_file.gets)
+            tokens = line.chomp.split("\t")
+            @@mappings[tokens[1]] = tokens[0].to_f
+          end
+
+          word_file.close
+        end
+      end
+    end
+  end
+end

data/lib/sentiment_lib/analysis/strategies/financial_dict_strategy.rb

@@ -0,0 +1,63 @@
+require File.expand_path('../../strategy', __FILE__)
+
+module SentimentLib
+  module Analysis
+    module Strategies
+      class FinancialDictStrategy < ::SentimentLib::Analysis::Strategy
+        @@mappings = nil
+
+        def initialize
+          self.class._load_mappings if !@@mappings
+        end
+
+        def mappings
+          @@mappings
+        end
+
+        # weight is pure sum (not average)
+        def weigh(tokens, opts={})
+          valid_tokens = tokens.delete_if { |token|
+            mappings[token] == nil
+          }
+
+          return 0 if valid_tokens.length == 0
+
+          sum = 0
+
+          valid_tokens.each { |token| 
+            sum += mappings[token]
+          }
+
+          if opts[:average] == true
+            sum / valid_tokens.length
+          else
+            sum
+          end
+        end          
+
+        private
+
+        def self._load_mappings
+          @@mappings = {}
+
+          data_root = File.expand_path('../../../../sentiment_lib/data', __FILE__)
+          
+          positive_word_file = File.new("#{data_root}/analysis/financial_dict/positive.csv", 'r')
+          negative_word_file = File.new("#{data_root}/analysis/financial_dict/negative.csv", 'r')
+
+          while (line = positive_word_file.gets)
+            tokens = line.chomp.split(",")
+            @@mappings[tokens[0].downcase] = 1.0
+          end
+
+          while (line = negative_word_file.gets)
+            tokens = line.chomp.split(",")
+            @@mappings[tokens[0].downcase] = -1.0
+          end            
+
+          [positive_word_file, negative_word_file].map &:close
+        end
+      end
+    end
+  end
+end

data/lib/sentiment_lib/analysis/strategy.rb

@@ -0,0 +1,34 @@
+module SentimentLib
+  module Analysis
+    class Strategy 
+
+      def normalize(str)
+        str.gsub(/[^[:alnum:]]/, ' ').downcase
+      end
+
+      def tokenize(str)
+        str.split
+      end 
+
+      # default weight is an average of per-word
+      # weight.  Optionally override in derived strategies.
+      def weigh(tokens)
+        valid_tokens = tokens.delete_if { |token|
+          mappings[token] == nil
+        }
+
+        return 0 if valid_tokens.length == 0
+
+        sum = 0
+
+        valid_tokens.each { |token| 
+          sum += mappings[token]
+        }
+
+        avg = sum / valid_tokens.length
+
+        avg          
+      end
+    end
+  end
+end

data/lib/sentiment_lib/analyzer.rb

@@ -0,0 +1,17 @@
+Dir[File.expand_path("../analysis/strategies/**/*.rb", __FILE__)].each { |f| require f }
+
+module SentimentLib
+  class Analyzer
+    attr_accessor :strategy
+
+    def initialize(opts={})
+      @strategy = opts[:strategy] || ::SentimentLib::Analysis::Strategies::FinancialDictStrategy.new
+    end
+
+    def analyze(str, opts={})
+      tokens, sum = @strategy.tokenize(@strategy.normalize(str)), 0
+
+      @strategy.weigh(tokens)
+    end
+  end
+end

data/lib/sentiment_lib/data/analysis/basic_dict/symbols.txt

@@ -0,0 +1,55 @@
+-1.0	%-(
+-1.0	)-:
+-1.0	):
+-1.0	)o:
+-1.0	8-0
+-1.0	8/
+-1.0	8\
+-1.0	8c
+-1.0	:'(
+-1.0	:'-(
+-1.0	:(
+-1.0	:*(
+-1.0	:,(
+-1.0	:-(
+-1.0	:-/
+-1.0	:-S
+-1.0	:-\
+-0.50	:-|
+-0.50	:/
+-0.25	:O
+-0.25	:S
+-0.25	:\
+-0.25	:|
+-1.0	=(
+-1.0	>:(
+-1.0	D:
+-1.0	sux
+1.0	(o;
+1.00	8-)
+1.0	;)
+1.0	;o)
+1.0	%-)
+1.0	(-:
+1.0	(:
+1.0	(o:
+1.0	8)
+1.0	:)
+1.0	:-D
+1.0	:-P
+1.0	:D
+1.0	:P
+1.0	:P
+1.0	:]
+1.0	:o)
+1.0	:p
+1.0	;^)
+1.0	<3
+1.0	&lt;3
+1.0	=)
+1.0	=]
+1.0	>:)
+1.0	>:D
+1.0	>=D
+1.0	^_^
+1.0	}:)