text_rank 1.2.4 → 1.3.1

data/ext/text_rank/page_rank_sparse_native.c

@@ -0,0 +1,293 @@
+#include <ruby.h>
+#include <math.h>
+#include <page_rank_sparse_native.h>
+
+const size_t NODE_LIST_SIZE = sizeof(NodeListStruct);
+const size_t EDGE_LIST_SIZE = sizeof(EdgeListStruct);
+const size_t NODE_SIZE = sizeof(NodeStruct);
+const size_t EDGE_SIZE = sizeof(EdgeStruct);
+const size_t GRAPH_SIZE = sizeof(GraphStruct);
+
+static const rb_data_type_t graph_typed_data = {
+  "PageRank/SparseNative/Graph",
+  { 0, free_graph, },
+  0, 0,
+  RUBY_TYPED_FREE_IMMEDIATELY,
+};
+
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void Init_sparse_native() {
+  VALUE PageRankModule, SparseNativeClass;
+
+  PageRankModule = rb_const_get(rb_cObject, rb_intern("PageRank"));
+  SparseNativeClass = rb_const_get(PageRankModule, rb_intern("SparseNative"));
+
+  rb_define_alloc_func(SparseNativeClass, sparse_native_allocate);
+  rb_define_private_method(SparseNativeClass, "_add_edge", sparse_native_add_edge, 3);
+  rb_define_private_method(SparseNativeClass, "_calculate", sparse_native_calculate, 3);
+}
+
+VALUE sparse_native_allocate(VALUE self) {
+  Graph g = malloc(GRAPH_SIZE);
+
+  // Grab a reference to the hash type used by a generic Ruby {}
+  // which accepts any key and any value.  We'll need this type to create
+  // a st_table in which to put arbitrary VALUE keys.  This hash type
+  // should be a static constant and thus should be safe to utilize without
+  // fear of garbage collection.
+  const struct st_hash_type *objhash = rb_hash_tbl(rb_hash_new(), "page_rank_sparse_native.c", 40)->type;
+
+  g->node_count = 0;
+  g->nodes = NULL;
+  g->dangling_nodes = NULL;
+  g->node_lookup = st_init_table_with_size(objhash, 0);
+
+  return TypedData_Wrap_Struct(self, &graph_typed_data, g);
+}
+
+VALUE sparse_native_add_edge(VALUE self, VALUE source, VALUE dest, VALUE weight) {
+  Graph g;
+
+  TypedData_Get_Struct(self, GraphStruct, &graph_typed_data, g);
+  add_edge_with_labels(g, source, dest, NUM2DBL(weight));
+  return Qnil;
+}
+
+VALUE sparse_native_calculate(VALUE self, VALUE max_iterations, VALUE damping, VALUE tolerance) {
+  Graph g;
+  VALUE ranks;
+
+  TypedData_Get_Struct(self, GraphStruct, &graph_typed_data, g);
+  calculate(g, FIX2INT(max_iterations), NUM2DBL(damping), NUM2DBL(tolerance));
+
+  ranks = rb_hash_new();
+  sort_and_normalize_ranks(g, rb_hash_dset, ranks);
+  return ranks;
+}
+
+void rb_hash_dset(VALUE hash, VALUE key, double value) {
+  rb_hash_aset(hash, key, DBL2NUM(value));
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void free_graph(void *data) {
+  Graph g = (Graph)data;
+  free_node_list(g->nodes, free_node);
+  free_node_list(g->dangling_nodes, NULL);
+  free(g->node_lookup);
+  free(g);
+}
+
+void free_node(Node n) {
+  free_edge_list(n->source_edges, free_edge);
+  free(n);
+}
+
+void free_node_list(NodeList nodes, void (*free_item)(Node)) {
+  while (nodes != NULL) {
+    NodeList tmp = nodes;
+    nodes = nodes->next;
+    if (free_item) {
+      free_item(tmp->node);
+    }
+    free(tmp);
+  }
+}
+
+void free_edge(Edge e) {
+  // Assume source node was allocated elsewhere and will be free'd elsewhere
+  free(e);
+}
+
+void free_edge_list(EdgeList edges, void (*free_item)(Edge)) {
+  while (edges != NULL) {
+    EdgeList tmp = edges;
+    edges = edges->next;
+    if (free_item) {
+      free_item(tmp->edge);
+    }
+    free(tmp);
+  }
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+Node add_node(Graph g, VALUE label) {
+  NodeList tmp = malloc(NODE_LIST_SIZE);
+
+  tmp->node = malloc(NODE_SIZE);
+  tmp->node->label = label;
+  tmp->node->source_edges = NULL;
+  tmp->node->rank = 0.0;
+  tmp->node->prev_rank = 0.0;
+  tmp->node->outbound_weight_total = 0.0;
+
+  tmp->next = g->nodes;
+  g->nodes = tmp;
+  g->node_count += 1;
+
+  return tmp->node;
+}
+
+Node add_dangling_node(Graph g, Node n) {
+  NodeList tmp = malloc(NODE_LIST_SIZE);
+
+  tmp->node = n;
+  tmp->next = g->dangling_nodes;
+  g->dangling_nodes = tmp;
+
+  return n;
+}
+
+Edge add_edge(Node source, Node destination, double weight) {
+  EdgeList tmp = malloc(EDGE_LIST_SIZE);
+
+  tmp->edge = malloc(EDGE_SIZE);
+  tmp->edge->source = source;
+  tmp->edge->weight = weight;
+
+  tmp->next = destination->source_edges;
+  destination->source_edges = tmp;
+  source->outbound_weight_total += weight;
+
+  return tmp->edge;
+}
+
+Edge add_edge_with_labels(Graph g, VALUE source_label, VALUE dest_label, double weight) {
+  Node source, dest;
+
+  source = lookup_node(g, source_label);
+  dest = lookup_node(g, dest_label);
+
+  return add_edge(source, dest, weight);
+}
+
+Node lookup_node(Graph g, VALUE label) {
+  Node n;
+
+  if (!st_lookup(g->node_lookup, (st_data_t)label, (st_data_t *)&n)) {
+    n = add_node(g, label);
+    st_add_direct(g->node_lookup, (st_data_t)label, (st_data_t)n);
+  }
+  return n;
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void calculate_start(Graph g) {
+  NodeList nodes;
+  Node source, destination;
+  EdgeList edges;
+  Edge e;
+
+  for (nodes = g->nodes; nodes != NULL; nodes = nodes->next) {
+    destination = nodes->node;
+
+    // If there is no outband, this is a "dangling" node
+    if (destination->outbound_weight_total == 0.0) {
+      add_dangling_node(g, destination);
+    }
+
+    // Normalize all source edge weights
+    for (edges = destination->source_edges; edges != NULL; edges = edges->next) {
+      e = edges->edge;
+      source = e->source;
+      e->weight = e->weight / source->outbound_weight_total;
+    }
+
+    // Set the initial rank
+    destination->prev_rank = 0;
+    destination->rank = 1.0 / g->node_count;
+  }
+}
+
+void calculate_step(Graph g, double damping) {
+  NodeList nodes, dangling_nodes;
+  Node source, destination;
+  EdgeList edges;
+  Edge e;
+
+  // Set prev rank to rank for all nodes
+  for (nodes = g->nodes; nodes != NULL; nodes = nodes->next) {
+    destination = nodes->node;
+    destination->prev_rank = destination->rank;
+  }
+
+  // Re-destribute the rankings according to weight
+  for (nodes = g->nodes; nodes != NULL; nodes = nodes->next) {
+    destination = nodes->node;
+    double sum = 0.0;
+    for (edges = destination->source_edges; edges != NULL; edges = edges->next) {
+      e = edges->edge;
+      source = e->source;
+      sum += source->prev_rank * e->weight;
+    }
+    for (dangling_nodes = g->dangling_nodes; dangling_nodes != NULL; dangling_nodes = dangling_nodes->next) {
+      source = dangling_nodes->node;
+      sum += source->prev_rank / g->node_count;
+    }
+    destination->rank = damping * sum + (1 - damping) / g->node_count;
+  }
+}
+
+// Calculate the Euclidean distance from prev_rank to rank across all nodes
+double prev_distance(Graph g) {
+  double sum_squares = 0.0;
+
+  for (NodeList nodes = g->nodes; nodes != NULL; nodes = nodes->next) {
+    Node n = nodes->node;
+    double rank_diff = n->prev_rank - n->rank;
+    sum_squares += rank_diff * rank_diff;
+  }
+
+  return sqrt(sum_squares);
+}
+
+void calculate(Graph g, int max_iterations, double damping, double tolerance) {
+  calculate_start(g);
+
+  while (max_iterations != 0) { // If negative one, allow to go without limit
+    calculate_step(g, damping);
+    if (prev_distance(g) < tolerance) {
+      break;
+    }
+    max_iterations--;
+  }
+}
+
+int node_compare(const void *v1, const void *v2) {
+  double rank1 = (*(Node *)v1)->rank;
+  double rank2 = (*(Node *)v2)->rank;
+  double cmp = rank2 - rank1; // Decreasing order
+  if (cmp < 0) return -1;
+  if (cmp > 0) return 1;
+  return 0;
+}
+
+void sort_and_normalize_ranks(Graph g, void (*callback)(VALUE, VALUE, double), VALUE callback_arg) {
+  NodeList nodes;
+  Node n;
+  double sum = 0.0;
+  unsigned long i;
+  Node *tmp;
+
+  i = g->node_count;
+  tmp = malloc(g->node_count * sizeof(Node));
+  for (nodes = g->nodes; nodes != NULL; nodes = nodes->next) {
+    n = nodes->node;
+    tmp[--i] = n;
+    sum += n->rank;
+  }
+
+  qsort(tmp, g->node_count, sizeof(Node), node_compare);
+
+  for (i = 0; i < g->node_count; i++) {
+    n = tmp[i];
+    callback(callback_arg, n->label, n->rank / sum);
+  }
+
+  free(tmp);
+}

data/ext/text_rank/page_rank_sparse_native.h

@@ -0,0 +1,93 @@
+#ifndef PAGE_RANK_SPARSE_NATIVE_H
+#define PAGE_RANK_SPARSE_NATIVE_H
+
+#include <ruby.h>
+
+struct NodeListStruct;
+typedef struct NodeListStruct* NodeList;
+
+typedef struct NodeListStruct {
+  struct NodeStruct *node;
+  struct NodeListStruct *next;
+} NodeListStruct;
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+struct EdgeListStruct;
+typedef struct EdgeListStruct* EdgeList;
+
+typedef struct EdgeListStruct {
+  struct EdgeStruct *edge;
+  struct EdgeListStruct *next;
+} EdgeListStruct;
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+struct NodeStruct;
+typedef struct NodeStruct* Node;
+
+typedef struct NodeStruct {
+  EdgeList source_edges;
+  VALUE label;
+  double prev_rank;
+  double rank;
+  double outbound_weight_total;
+} NodeStruct;
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+struct EdgeStruct;
+typedef struct EdgeStruct* Edge;
+
+typedef struct EdgeStruct {
+  Node source;
+  double weight;
+} EdgeStruct;
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+struct GraphStruct;
+typedef struct GraphStruct* Graph;
+
+typedef struct GraphStruct {
+  unsigned long node_count;
+  NodeList nodes;
+  NodeList dangling_nodes;
+  st_table *node_lookup;
+} GraphStruct;
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void free_graph(void *data);
+void free_node(Node n);
+void free_node_list(NodeList nodes, void (*free_item)(Node));
+void free_edge(Edge e);
+void free_edge_list(EdgeList edges, void (*free_item)(Edge));
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+Node add_node(Graph g, VALUE label);
+Node add_dangling_node(Graph g, Node n);
+Edge add_edge(Node source, Node destination, double weight);
+Edge add_edge_with_labels(Graph g, VALUE source_label, VALUE dest_label, double weight);
+Node lookup_node(Graph g, VALUE label);
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void calculate_start(Graph g);
+void calculate_step(Graph g, double damping);
+double prev_distance(Graph g);
+void calculate(Graph g, int max_iterations, double damping, double tolerance);
+int node_compare(const void *v1, const void *v2);
+void sort_and_normalize_ranks(Graph g, void (*callback)(VALUE, VALUE, double), VALUE callback_arg);
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void Init_sparse_native();
+VALUE sparse_native_allocate(VALUE self);
+VALUE sparse_native_add_edge(VALUE self, VALUE source, VALUE dest, VALUE weight);
+VALUE sparse_native_calculate(VALUE self, VALUE max_iterations, VALUE damping, VALUE tolerance);
+VALUE sorted_and_normalized_ranks(Graph g);
+void rb_hash_dset(VALUE hash, VALUE key, double value);
+
+#endif

data/ext/text_rank/text_rank.c

@@ -0,0 +1,6 @@
+#include <page_rank_sparse_native.h>
+
+// cppcheck-suppress unusedFunction
+void Init_text_rank() {
+  Init_sparse_native();
+}

data/lib/page_rank/base.rb

@@ -7,6 +7,8 @@ module PageRank
   ##
   class Base
 
+    attr_reader :damping, :tolerance
+
     # @param (see #damping=)
     # @param (see #tolerance=)
     def initialize(damping: nil, tolerance: nil, **_)
@@ -48,7 +50,7 @@ module PageRank
 
         prev_ranks = ranks
         ranks = calculate_step(ranks)
-        break if distance(ranks, prev_ranks) < @tolerance
+        break if distance(ranks, prev_ranks) < tolerance
 
         max_iterations -= 1
       end
@@ -81,7 +83,7 @@ module PageRank
     def distance(vector1, vector2)
       sum_squares = node_count.times.reduce(0.0) do |sum, i|
         d = vector1[i] - vector2[i]
-        sum + d * d
+        sum + (d * d)
       end
       Math.sqrt(sum_squares)
     end

data/lib/page_rank/dense.rb

@@ -79,7 +79,7 @@ module PageRank
         total = total_out_weights[source_idx]
         if total
           w = @out_links[source_idx][dest_idx] || 0.0
-          @damping * w / total + (1 - @damping) / node_count.to_f
+          (damping * w / total) + ((1 - damping) / node_count.to_f)
         else
           1.0 / node_count.to_f
         end

data/lib/page_rank/sparse.rb

@@ -56,7 +56,7 @@ module PageRank
           w / @weight_totals[source]
         end
       end
-      Hash[@nodes.map { |k| [k, 1.0 / node_count.to_f] }]
+      @nodes.to_h { |k| [k, 1.0 / node_count.to_f] }
     end
 
     def calculate_step(ranks)
@@ -68,14 +68,14 @@ module PageRank
         @dangling_nodes.each do |source|
           sum += ranks[source] / node_count.to_f
         end
-        new_ranks[dest] = @damping * sum + (1 - @damping) / node_count
+        new_ranks[dest] = (damping * sum) + ((1 - damping) / node_count)
       end
     end
 
     def sort_ranks(ranks)
       sum = 0.0
       ranks.each { |_, v| sum += v }
-      Hash[ranks.map { |k, v| [k, v / sum] }.sort_by { |_, v| -v }]
+      ranks.map { |k, v| [k, v / sum] }.sort_by { |_, v| -v }.to_h
     end
 
     def distance(vector1, vector2)

data/lib/page_rank/sparse_native.rb

@@ -0,0 +1,21 @@
+module PageRank
+  class SparseNative < Base
+
+    # require 'page_rank/sparse_native.so'
+
+    # @param (see Base#add)
+    # @param weight [Float] Optional weight for the graph edge
+    # @return (see Base#add)
+    def add(source, dest, weight: 1.0)
+      _add_edge(source, dest, weight) unless source == dest
+    end
+
+    # Perform the PageRank calculation
+    # @param max_iterations [Fixnum] Maximum number of PageRank iterations to perform (or -1 for no max)
+    # @return [Hash<Object, Float>] of nodes with rank
+    def calculate(max_iterations: -1, **_)
+      _calculate(max_iterations, damping, tolerance)
+    end
+
+  end
+end

data/lib/page_rank.rb

@@ -17,16 +17,17 @@ require 'set'
 ##
 module PageRank
 
-  autoload :Base,   'page_rank/base'
-  autoload :Dense,  'page_rank/dense'
-  autoload :Sparse, 'page_rank/sparse'
+  autoload :Base,         'page_rank/base'
+  autoload :Dense,        'page_rank/dense'
+  autoload :Sparse,       'page_rank/sparse'
+  autoload :SparseNative, 'page_rank/sparse_native'
 
   # @option options [Symbol] :strategy PageRank strategy to use (either :sparse or :dense)
   # @option options [Float]  :damping The probability of following the graph vs. randomly choosing a new node
   # @option options [Float]  :tolerance The desired accuracy of the results
   # @return [PageRank::Base]
   def self.new(strategy: :sparse, **options)
-    const_get(strategy.to_s.capitalize).new(**options)
+    const_get(strategy.to_s.split('_').map(&:capitalize).join).new(**options)
   end
 
   # Convenience method to quickly calculate PageRank. In the calling block, graph edges can be added.

data/lib/text_rank/char_filter/ascii_folding.rb

@@ -3,14 +3,12 @@ module TextRank
     ##
     # Characater filter to transform non-ASCII (unicode) characters into ASCII-friendly versions.
     #
-    # rubocop:disable Style/AsciiComments
     #
     # = Example
     #
     #  AsciiFolding.new.filter!("the Perigordian Abbé then made answer, because a poor beggar of the country of Atrébatie heard some foolish things said")
     #  => "the Perigordian Abbe then made answer, because a poor beggar of the country of Atrebatie heard some foolish things said"
     #
-    # rubocop:enable Style/AsciiComments
     #
     ##
     class AsciiFolding

data/lib/text_rank/char_filter/strip_html.rb

@@ -17,6 +17,7 @@ module TextRank
     class StripHtml < Nokogiri::XML::SAX::Document
 
       def initialize
+        super
         @text = StringIO.new
       end
 

data/lib/text_rank/fingerprint.rb

@@ -57,7 +57,7 @@ module TextRank
     end
 
     # Calculates the "similarity" between this fingerprint and another
-    # @param {Fingerprint} A second fingerprint to compare
+    # @param {Fingerprint} other A second fingerprint to compare
     # @return [Number] A number between 0.0 (different) and 1.0 (same)
     def similarity(other)
       return 1.0 if values == other.values # Short-circuit for efficiency
@@ -83,7 +83,7 @@ module TextRank
 
     def norm_factor
       @norm_factor ||= size.times.reduce(0.0) do |s, i|
-        s + (i + 1) / Math.log(i + 2) / size.to_f
+        s + ((i + 1) / Math.log(i + 2) / size.to_f)
       end
     end
 

data/lib/text_rank/graph_strategy/coocurrence.rb

@@ -60,7 +60,7 @@ module TextRank
       # @param graph [PageRank::Base] a PageRank graph into which to add nodes/edges
       # return [nil]
       def build_graph(tokens, graph)
-        ngram_window = @ngram_size * 2 + 1
+        ngram_window = (@ngram_size * 2) + 1
         tokens.size.times do |i|
           ngram_window.times do |j|
             consider_ngram_window(tokens, graph, i, j)
@@ -71,14 +71,14 @@ module TextRank
 
       private
 
-      def consider_ngram_window(tokens, graph, i, j)
-        return if j == @ngram_size || i + j < @ngram_size
+      def consider_ngram_window(tokens, graph, idx_i, idx_j)
+        return if idx_j == @ngram_size || idx_i + idx_j < @ngram_size
 
-        token_i = tokens[i]
-        token_j = tokens[i - @ngram_size + j]
+        token_i = tokens[idx_i]
+        token_j = tokens[idx_i - @ngram_size + idx_j]
 
         if token_j
-          graph.add(token_i, token_j, weight: 1.0 / (j - @ngram_size).abs)
+          graph.add(token_i, token_j, weight: 1.0 / (idx_j - @ngram_size).abs)
         end
       end
 

data/lib/text_rank/keyword_extractor.rb

@@ -71,7 +71,6 @@ module TextRank
     end
 
     # Sets the graph strategy for producing a graph from tokens
-    # @param strategy [Class, Symbol, #build_graph] Strategy for producing a graph from tokens
     # @return [Class, Symbol, #build_graph]
     attr_writer :graph_strategy
 
@@ -103,14 +102,23 @@ module TextRank
     end
 
     # Filter & tokenize text, and return PageRank
-    # @param text [String] unfiltered text to be processed
+    # @param text [String,Array<String>] unfiltered text to be processed
     # @return [Hash<String, Float>] tokens and page ranks (in descending order)
     def extract(text, **options)
-      tokens = tokenize(text)
+      text = Array(text)
+      tokens_per_text = text.map do |t|
+        tokenize(t)
+      end
       graph = PageRank.new(**@page_rank_options)
-      classify(@graph_strategy, context: GraphStrategy).build_graph(tokens, graph)
+      strategy = classify(@graph_strategy, context: GraphStrategy)
+      tokens_per_text.each do |tokens|
+        strategy.build_graph(tokens, graph)
+      end
       ranks = graph.calculate(**options)
-      apply_rank_filters(ranks, tokens: tokens, original_text: text)
+      tokens_per_text.each_with_index do |tokens, i|
+        ranks = apply_rank_filters(ranks, tokens: tokens, original_text: text[i])
+      end
+      ranks
     end
 
     private

data/lib/text_rank/rank_filter/collapse_adjacent.rb

@@ -151,7 +151,7 @@ module TextRank
         # tokenization (e.g. ASCII folding).  That's okay.  We're just making the best effort we can
         # to find what we can.
         def scan_text_for_all_permutations_of(single_tokens)
-          # NOTE that by reversing the order we craft the regex to prefer larger combinations over
+          # NOTE: that by reversing the order we craft the regex to prefer larger combinations over
           # smaller combinations (or singletons).
           perms = (1..@max_tokens_to_combine).to_a.reverse.flat_map do |n|
             scan_text_for_n_permutations_of(single_tokens, n)
@@ -162,8 +162,8 @@ module TextRank
           end unless perms.empty?
         end
 
-        def scan_text_for_n_permutations_of(single_tokens, n)
-          single_tokens.permutation(n).map do |perm|
+        def scan_text_for_n_permutations_of(single_tokens, n_perms)
+          single_tokens.permutation(n_perms).map do |perm|
             unless @permutations_scanned.key?(perm)
               @permutations_scanned[perm] = 0
               perm

data/lib/text_rank/rank_filter/sort_by_value.rb

@@ -14,7 +14,7 @@ module TextRank
       # @param ranks [Hash<String, Float>] the results of the PageRank algorithm
       # @return [Hash<String, Float>]
       def filter!(ranks, **_)
-        Hash[ranks.sort_by { |_, v| @descending ? -v : v }]
+        ranks.sort_by { |_, v| @descending ? -v : v }.to_h
       end
 
     end

data/lib/text_rank/tokenizer/money.rb

@@ -1,7 +1,7 @@
 module TextRank
   module Tokenizer
 
-    CURRENCY_SYMBOLS = '[' + [
+    CURRENCY_SYMBOLS = "[#{[
       "\u00a4", # Generic Currency Symbol
       "\u0024", # Dollar Sign
       "\u00a2", # Cent Sign
@@ -26,14 +26,13 @@ module TextRank
       "\u20ab", # Dong Sign
       "\u0025", # Percent
       "\u2030", # Per Million
-    ].join + ']'
+    ].join}]"
     private_constant :CURRENCY_SYMBOLS # Do not expose this to avoid confusion
 
     ##
     # A tokenizer regex that preserves money or formatted numbers as a single token. This
     # currently supports 24 different currency symbols:
     #
-    # rubocop:disable Style/AsciiComments
     #
     # * ¤
     # * $
@@ -60,7 +59,6 @@ module TextRank
     # * %
     # * ‰
 
-    # rubocop:enable Style/AsciiComments
     #
     # It also supports two alternative formats for negatives as well as optional three digit comma
     # separation and optional decimals.