graph-rank 0.0.1

data/LICENSE

@@ -0,0 +1,16 @@
+A Ruby implementation of the TextRank and PageRank algorithms.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Author: Louis-Antoine Mullie (louis.mullie@gmail.com). Copyright 2012.

data/README.md

@@ -0,0 +1,71 @@
+###About
+
+This gem implements a PageRank class and a class that allows to perform keyword ranking using the TextRank algorithm. Both were ported from the [PHP Implementation](https://github.com/crodas/textrank) by @crodas.
+
+###Install
+
+```
+gem install graph-rank
+```
+
+###Usage
+
+**TextRank**
+
+```ruby
+text = 'PageRank is a link analysis algorithm, named after Larry ' +
+'Page and used by the Google Internet search engine, that assigns ' +
+'a numerical weighting to each element of a hyperlinked set of ' +
+'documents, such as the World Wide Web, with the purpose of "measuring"' +
+'its relative importance within the set.'
+
+tr = GraphRank::Keywords.new
+
+tr.run(text)
+
+```
+
+Optionally, you can pass the n-gram size (default = 3), as well as the damping and convergence (see PageRank) to the constructor. Finally, you can set stop words as follows:
+
+```ruby
+t.stop_words = ["word", "another", "etc"]
+```
+
+The default stop word list is as follows:
+
+    "about","also","are","away","because",
+    "been","beside","besides","between","but","cannot",
+    "could","did","etc","even","ever","every","for","had",
+    "have","how","into","isn","maybe","non","nor","now",
+    "should","such","than","that","then","these","this",
+    "those","though","too","was","wasn","were","what","when",
+    "where","which","while","who","whom","whose","will",
+    "with","would","wouldn","yes"
+
+> Reference: R. Mihalcea and P. Tarau, “TextRank: Bringing Order into Texts,” in Proceedings of EMNLP 2004. Association for Computational Linguistics, 2004, pp. 404–411.
+
+**PageRank**
+
+```ruby
+
+pr = GraphRank::PageRank.new
+
+pr.add(1,2)
+pr.add(1,4)
+pr.add(1,5)
+pr.add(4,5)
+pr.add(4,1)
+pr.add(4,3)
+pr.add(1,3)
+pr.add(3,1)
+pr.add(5,1)
+
+pr.calculate
+# => [[1, 5.99497754810465], [3, 2.694723988738302], 
+#    [5, 2.694723988738302], [4, 2.100731029131304],
+#    [2, 2.100731029131304]]
+```
+
+Optionally, you can pass the damping factor (default = 0.85) and the convergence criterion (default = 0.01) as parameters to the PageRank constructor.
+
+> Reference: Brin, S.; Page, L. (1998). "The anatomy of a large-scale hypertextual Web search engine". Computer Networks and ISDN Systems 30: 107–117.

data/lib/graph-rank.rb

@@ -0,0 +1,13 @@
+module GraphRank
+  
+  # Version number.
+  VERSION = '0.0.1'
+  
+  # Core classes.
+  require 'graph-rank/page_rank'
+  require 'graph-rank/text_rank'
+  
+  # Implementations.
+  require 'graph-rank/keywords'
+  
+end

data/lib/graph-rank/keywords.rb

@@ -0,0 +1,39 @@
+# Implement the PageRank algorithm
+# for unsupervised keyword extraction.
+class GraphRank::Keywords < GraphRank::TextRank
+
+  # Split the text on words.
+  def get_features
+    @text.split(' ')
+  end
+
+  # Remove short and stop words.
+  def filter_features
+    remove_short_words
+    remove_stop_words
+  end
+
+  # Remove all stop words.
+  def remove_stop_words
+    @features.delete_if { |word| @stop_words.include?(word) }
+  end
+
+  # Remove 1 and 2 char words.
+  def remove_short_words
+    @features.delete_if { |word| word.length < 3 }
+  end
+
+  # Build the co-occurence graph for an n-gram.
+  def build_graph
+    @features.each_with_index do |f,i|
+      min, max = i - @ngram_size, i + @ngram_size
+      while min < max
+        if @features[min] and min != i
+          @ranking.add(@features[i], @features[min])
+        end
+        min += 1
+      end
+    end
+  end
+
+end

data/lib/graph-rank/page_rank.rb

@@ -0,0 +1,68 @@
+# Brin, S.; Page, L. (1998). "The anatomy of 
+# a large-scale hypertextual Web search engine". 
+# Computer Networks and ISDN Systems 30: 107–117.
+class GraphRank::PageRank
+
+  # Initialize with default damping and convergence.
+  def initialize(damping=nil, convergence=nil)
+    damping ||= 0.85; convergence ||= 0.01
+    if damping <= 0 or damping > 1
+      raise 'Invalid damping factor.'
+    elsif convergence < 0 or convergence > 1
+      raise 'Invalid convergence factor.'
+    end
+    @damping, @convergence = damping, convergence
+    @graph, @outlinks, @nodes = {}, {}, {}  ####
+  end
+
+  # Add a node to the graph.
+  def add(source, dest)
+    return false if source == dest
+    @outlinks[source] ||= 0
+    @graph[dest] ||= []
+    @graph[dest] << source
+    @outlinks[source] += 1
+    @nodes[source] = 0.15
+    @nodes[dest] = 0.15
+  end
+
+  # Iterates the PageRank algorithm
+  # until convergence is reached.
+  def calculate
+    done = false
+    until done
+      new_nodes = iteration
+      done = convergence(new_nodes)
+      @nodes = new_nodes
+    end
+    @nodes.sort_by {|k,v|v}.reverse
+  end
+
+  private
+
+  # Performs one iteration to calculate
+  # the PageRank ranking for all nodes.
+  def iteration
+    new_nodes = {}
+    @graph.each do |node,links|
+      score = links.map do |id|
+        @nodes[id] / @outlinks[id]
+      end.inject(:+)
+      new_nodes[node] = (1-@damping/
+      @nodes.size) + @damping * score
+    end
+    new_nodes
+  end
+
+  # Check for convergence.
+  def convergence(current)
+    diff = {}
+    @nodes.each do |k,v|
+      diff[k] = current[k] - @nodes[k]
+    end
+    total = 0
+    diff.each { |k,v| total += diff[k] * v }
+    Math.sqrt(total/current.size) < @convergence
+  end
+
+end

data/lib/graph-rank/text_rank.rb

@@ -0,0 +1,56 @@
+# R. Mihalcea and P. Tarau, “TextRank: Bringing Order into Texts,”
+# in Proceedings of EMNLP 2004. Association for Computational
+# Linguistics, 2004, pp. 404–411.
+class GraphRank::TextRank
+
+  # Default English stop-word list.
+  StopWords = ["a", "about", "above", "above", "across", "after", "afterwards", "again", "against", "all", "almost", "alone", "along", "already", "also","although","always","am","among", "amongst", "amoungst", "amount",  "an", "and", "another", "any","anyhow","anyone","anything","anyway", "anywhere", "are", "around", "as",  "at", "back","be","became", "because","become","becomes", "becoming", "been", "before", "beforehand", "behind", "being", "below", "beside", "besides", "between", "beyond", "bill", "both", "bottom","but", "by", "call", "can", "cannot", "cant", "co", "con", "could", "couldnt", "cry", "de", "describe", "detail", "do", "done", "down", "due", "during", "each", "eg", "eight", "either", "eleven","else", "elsewhere", "empty", "enough", "etc", "even", "ever", "every", "everyone", "everything", "everywhere", "except", "few", "fifteen", "fify", "fill", "find", "fire", "first", "five", "for", "former", "formerly", "forty", "found", "four", "from", "front", "full", "further", "get", "give", "go", "had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter", "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his", "how", "however", "hundred", "ie", "if", "in", "inc", "indeed", "interest", "into", "is", "it", "its", "itself", "keep", "last", "latter", "latterly", "least", "less", "ltd", "made", "many", "may", "me", "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly", "move", "much", "must", "my", "myself", "name", "namely", "neither", "never", "nevertheless", "next", "nine", "no", "nobody", "none", "noone", "nor", "not", "nothing", "now", "nowhere", "of", "off", "often", "on", "once", "one", "only", "onto", "or", "other", "others", "otherwise", "our", "ours", "ourselves", "out", "over", "own","part", "per", "perhaps", "please", "put", "rather", "re", "same", "see", "seem", "seemed", "seeming", "seems", "serious", "several", "she", "should", "show", "side", "since", "sincere", "six", "sixty", "so", "some", "somehow", "someone", "something", "sometime", "sometimes", "somewhere", "still", "such", "system", "take", "ten", "than", "that", "the", "their", "them", "themselves", "then", "thence", "there", "thereafter", "thereby", "therefore", "therein", "thereupon", "these", "they", "thickv", "thin", "third", "this", "those", "though", "three", "through", "throughout", "thru", "thus", "to", "together", "too", "top", "toward", "towards", "twelve", "twenty", "two", "un", "under", "until", "up", "upon", "us", "very", "via", "was", "we", "well", "were", "what", "whatever", "when", "whence", "whenever", "where", "whereafter", "whereas", "whereby", "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", "who", "whoever", "whole", "whom", "whose", "why", "will", "with", "within", "without", "would", "yet", "you", "your", "yours", "yourself", "yourselves", "the"]
+
+  # Allow configuration of stop words.
+  attr_accessor :stop_words
+
+  # Initialize with ngram size, damping and convergence.
+  def initialize(ngram_size=3, damping=nil, convergence=nil)
+    @ranking = GraphRank::PageRank.new(damping, convergence)
+    @stop_words = StopWords
+    @ngram_size = ngram_size
+  end
+
+  # Add text and return PageRank.
+  def run(text)
+    @text = text
+    clean_text
+    @features = get_features
+    filter_features
+    build_graph
+    calculate_ranking
+  end
+
+  # Clean text leaving just letters from a-z.
+  def clean_text
+    @text.downcase!
+    @text.gsub!(/[^a-z ]/, ' ')
+    @text.gsub!(/\s+/, " ")
+  end
+
+  # Return the features (keyword, sentence, etc.)
+  def get_features
+    raise 'Must be implemented in subclass.'
+  end
+
+  # Return only features of interest.
+  def filter_features
+    @features
+  end
+
+  # Build the graph from the features.
+  def build_graph
+    raise 'Must be implemented in subclass.'
+  end
+
+  # Calculate the PageRank ranking.
+  def calculate_ranking
+    @ranking.calculate
+  end
+
+end

metadata

@@ -0,0 +1,51 @@
+--- !ruby/object:Gem::Specification
+name: graph-rank
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Louis Mullie
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-12-16 00:00:00.000000000 Z
+dependencies: []
+description: ! ' GraphRank is an impementation of TextRank and PageRank in Ruby. '
+email:
+- louis.mullie@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/graph-rank/keywords.rb
+- lib/graph-rank/page_rank.rb
+- lib/graph-rank/text_rank.rb
+- lib/graph-rank.rb
+- README.md
+- LICENSE
+homepage: https://github.com/louismullie/graphrank
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: ! 'GraphRank: bringing TextRank and PageRank to Ruby.'
+test_files: []