graph-rank 0.0.1 → 0.0.2

data/README.md

@@ -12,6 +12,8 @@ gem install graph-rank
 
 **TextRank**
 
+> Reference: R. Mihalcea and P. Tarau, “TextRank: Bringing Order into Texts,” in Proceedings of EMNLP 2004. Association for Computational Linguistics, 2004, pp. 404–411.
+
 ```ruby
 text = 'PageRank is a link analysis algorithm, named after Larry ' +
 'Page and used by the Google Internet search engine, that assigns ' +
@@ -21,31 +23,20 @@ text = 'PageRank is a link analysis algorithm, named after Larry ' +
 
 tr = GraphRank::Keywords.new
 
-tr.run(text)
+tr.run(text).inspect
 
 ```
 
 Optionally, you can pass the n-gram size (default = 3), as well as the damping and convergence (see PageRank) to the constructor. Finally, you can set stop words as follows:
 
 ```ruby
-t.stop_words = ["word", "another", "etc"]
+tr.stop_words = ["word", "another", "etc"]
 ```
 
-The default stop word list is as follows:
-
-    "about","also","are","away","because",
-    "been","beside","besides","between","but","cannot",
-    "could","did","etc","even","ever","every","for","had",
-    "have","how","into","isn","maybe","non","nor","now",
-    "should","such","than","that","then","these","this",
-    "those","though","too","was","wasn","were","what","when",
-    "where","which","while","who","whom","whose","will",
-    "with","would","wouldn","yes"
-
-> Reference: R. Mihalcea and P. Tarau, “TextRank: Bringing Order into Texts,” in Proceedings of EMNLP 2004. Association for Computational Linguistics, 2004, pp. 404–411.
-
 **PageRank**
 
+> Reference: Brin, S.; Page, L. (1998). "The anatomy of a large-scale hypertextual Web search engine". Computer Networks and ISDN Systems 30: 107–117.
+
 ```ruby
 
 pr = GraphRank::PageRank.new
@@ -60,12 +51,11 @@ pr.add(1,3)
 pr.add(3,1)
 pr.add(5,1)
 
-pr.calculate
+puts pr.calculate.inspect
+
 # => [[1, 5.99497754810465], [3, 2.694723988738302], 
 #    [5, 2.694723988738302], [4, 2.100731029131304],
 #    [2, 2.100731029131304]]
 ```
 
-Optionally, you can pass the damping factor (default = 0.85) and the convergence criterion (default = 0.01) as parameters to the PageRank constructor.
-
-> Reference: Brin, S.; Page, L. (1998). "The anatomy of a large-scale hypertextual Web search engine". Computer Networks and ISDN Systems 30: 107–117.
+Optionally, you can pass the damping factor (default = 0.85) and the convergence criterion (default = 0.01) as parameters to the PageRank constructor. Additionally, you can pass in an edge weight parameter to `#add` and it will be used in the PageRank calculation.

data/lib/graph-rank.rb

@@ -1,7 +1,7 @@
 module GraphRank
   
   # Version number.
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
   
   # Core classes.
   require 'graph-rank/page_rank'

data/lib/graph-rank/keywords.rb

@@ -4,7 +4,8 @@ class GraphRank::Keywords < GraphRank::TextRank
 
   # Split the text on words.
   def get_features
-    @text.split(' ')
+    clean_text
+    @features = @text.split(' ')
   end
 
   # Remove short and stop words.
@@ -13,6 +14,13 @@ class GraphRank::Keywords < GraphRank::TextRank
     remove_stop_words
   end
 
+  # Clean text leaving just letters from a-z.
+  def clean_text
+    @text.downcase!
+    @text.gsub!(/[^a-z ]/, ' ')
+    @text.gsub!(/\s+/, " ")
+  end
+
   # Remove all stop words.
   def remove_stop_words
     @features.delete_if { |word| @stop_words.include?(word) }

data/lib/graph-rank/page_rank.rb

@@ -12,18 +12,20 @@ class GraphRank::PageRank
       raise 'Invalid convergence factor.'
     end
     @damping, @convergence = damping, convergence
-    @graph, @outlinks, @nodes = {}, {}, {}  ####
+    @graph, @outlinks, @nodes, @weights = {}, {}, {}, {}
   end
 
   # Add a node to the graph.
-  def add(source, dest)
+  def add(source, dest, weight=1.0)
     return false if source == dest
-    @outlinks[source] ||= 0
+    @outlinks[source] ||= 0.0
     @graph[dest] ||= []
     @graph[dest] << source
-    @outlinks[source] += 1
+    @outlinks[source] += 1.0
     @nodes[source] = 0.15
     @nodes[dest] = 0.15
+    @weights[source] ||= {}
+    @weights[source][dest] = weight
   end
 
   # Iterates the PageRank algorithm
@@ -46,7 +48,7 @@ class GraphRank::PageRank
     new_nodes = {}
     @graph.each do |node,links|
       score = links.map do |id|
-        @nodes[id] / @outlinks[id]
+        @nodes[id] / @outlinks[id] * @weights[id][node]
       end.inject(:+)
       new_nodes[node] = (1-@damping/
       @nodes.size) + @damping * score
@@ -60,7 +62,7 @@ class GraphRank::PageRank
     @nodes.each do |k,v|
       diff[k] = current[k] - @nodes[k]
     end
-    total = 0
+    total = 0.0
     diff.each { |k,v| total += diff[k] * v }
     Math.sqrt(total/current.size) < @convergence
   end

data/lib/graph-rank/sentences.rb

@@ -0,0 +1,49 @@
+require 'graph-rank'
+# Implement the PageRank algorithm
+# for unsupervised sentence extraction.
+class GraphRank::Sentences < GraphRank::TextRank
+
+  begin
+    require 'treat'
+    include Treat::Core::DSL
+  rescue
+    puts "GraphRank::Sentences requires the treat " +
+    "gem to work. Please run `gem install treat`."
+  end
+
+  # Stem stop words!
+  def get_features
+    @section = section(@text)
+    .apply(:chunk,:segment,:tokenize)
+    @features = @section.groups
+  end
+  
+  # Build the co-occurence graph for an n-gram.
+  def build_graph
+    @features.each do |grp|
+      wc = grp.word_count
+      @features.each do |grp2|
+        wc2 = grp2.word_count
+        score = 0.0
+        grp.each_word do |wrd|
+          next if @stop_words.include?(wrd.to_s)
+          grp2.each_word do |wrd2|
+            score += 1 if wrd.stem == wrd2.stem
+          end
+        end
+        score /= (Math.log(wc) + Math.log(wc2))
+        @ranking.add(grp.id, grp2.id, score)
+      end
+    end
+  end
+  
+  def run(text,n=4)
+    rankings = super(text)
+    rankings = rankings[0..n].map { |x|x[0] }
+    @section.groups.select do |grp|
+      rankings.include?(grp.id)
+    end.map(&:to_s)
+  end
+  
+
+end

data/lib/graph-rank/text_rank.rb

@@ -19,20 +19,12 @@ class GraphRank::TextRank
   # Add text and return PageRank.
   def run(text)
     @text = text
-    clean_text
-    @features = get_features
+    get_features
     filter_features
     build_graph
     calculate_ranking
   end
 
-  # Clean text leaving just letters from a-z.
-  def clean_text
-    @text.downcase!
-    @text.gsub!(/[^a-z ]/, ' ')
-    @text.gsub!(/\s+/, " ")
-  end
-
   # Return the features (keyword, sentence, etc.)
   def get_features
     raise 'Must be implemented in subclass.'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: graph-rank
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -20,6 +20,7 @@ extra_rdoc_files: []
 files:
 - lib/graph-rank/keywords.rb
 - lib/graph-rank/page_rank.rb
+- lib/graph-rank/sentences.rb
 - lib/graph-rank/text_rank.rb
 - lib/graph-rank.rb
 - README.md