text_rank 1.2.3 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e1daece496ee8bb075ecc3540d6b8ca64d45c740b1ca34529e094283933d9c97
-  data.tar.gz: bee8ff77c21cffa95d838e91ae09773b290afd4b9c0224415c2d783f63069b1b
+  metadata.gz: 5b25779e7c013e9d0c1d867324f58d40a062bacae0e38f92714e1d3fd7b0e7ef
+  data.tar.gz: 34c36b8ff6673092b2463b9f4e0fdaf94a55e50c3e52e4aeec125775c7fa3a9d
 SHA512:
-  metadata.gz: 0e9df6c07d6c8bb94a782b61c7877e7fe2e4dd064645c87a0e636bae2236ffbf92e848426d5d8cb4daa04328a6afc25c0a58de3c14ba316c84bb07caa45801f8
-  data.tar.gz: e7a10407dce5651a05aa208db1136a3c0cc1082c70152b676d3867408cc8107dc8543080e20b8855659b8b58c4fe0063321fc5e8b976933372ff02d6b597ca82
+  metadata.gz: 6f03e71745ed96077c63ed376303fcfaa8683f960319d71a405b943aa4a23383938c914b33c867f76f4d979505aeb1d5a0110b51dcc1eadab14cab41d6ee8697
+  data.tar.gz: edb17a0ee101254a5afc7c7ee5b084e11ce1d8bfd5083d5069a5e0841751a1f8f175fa167b788186b881e72a1114fa17a8cddc8c3328e00dda812a89b8647196

data/.codeclimate.yml

@@ -16,7 +16,7 @@ engines:
     - 4218049e28199ed950d3cd721df86dce
     - c8179d0de3a9df18a2c45750d3f8647e
     - 03f6eee11d86507da564695007106721
-    channel: rubocop-0-85
+    channel: rubocop-1-23-0
 ratings:
   paths:
   - "**.rb"

data/.gitignore

@@ -8,3 +8,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+
+
+*.bundle
+*.so

data/.rubocop.yml

@@ -15,6 +15,10 @@ Layout/EmptyLinesAroundModuleBody:
 Layout/ExtraSpacing:
   Enabled: false
 
+Layout/HashAlignment:
+  EnforcedHashRocketStyle: table
+  EnforcedColonStyle: table
+
 Layout/LineLength:
   Max: 120
   Enabled: false
@@ -89,6 +93,9 @@ Style/GuardClause:
 Style/HashEachMethods:
   Enabled: true
 
+Style/HashSyntax:
+  Enabled: true
+
 Style/HashTransformKeys:
   Enabled: true
 

data/.ruby-version

@@ -1 +1 @@
-ruby-2.5.1
+ruby-3.0.3

data/.travis.yml

@@ -9,6 +9,7 @@ before_script:
   - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
   - chmod +x ./cc-test-reporter
   - ./cc-test-reporter before-build
+  - bundle exec rake compile
 script:
   - bundle exec rspec
 after_script:

data/Rakefile

@@ -1,4 +1,5 @@
 require "bundler/gem_tasks"
+require "rake/extensiontask"
 require "rspec/core/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
@@ -10,3 +11,7 @@ RDoc::Task.new do |rdoc|
   rdoc.main = "README.md"
   rdoc.rdoc_files.include("README.md", "lib/**/*.rb")
 end
+
+Rake::ExtensionTask.new('text_rank') do |ext|
+  ext.lib_dir = 'lib/text_rank'
+end

data/bin/console

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
-require "text_rank"
+require 'bundler/setup'
+require 'text_rank'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
@@ -10,5 +10,5 @@ require "text_rank"
 # require "pry"
 # Pry.start
 
-require "irb"
+require 'irb'
 IRB.start

data/ext/text_rank/extconf.rb

@@ -0,0 +1,3 @@
+require "mkmf"
+
+create_makefile('text_rank/text_rank')

data/ext/text_rank/page_rank_sparse_native.c

@@ -0,0 +1,300 @@
+#include <ruby.h>
+#include <math.h>
+#include <page_rank_sparse_native.h>
+
+const size_t NODE_LIST_SIZE = sizeof(NodeListStruct);
+const size_t EDGE_LIST_SIZE = sizeof(EdgeListStruct);
+const size_t NODE_SIZE = sizeof(NodeStruct);
+const size_t EDGE_SIZE = sizeof(EdgeStruct);
+const size_t GRAPH_SIZE = sizeof(GraphStruct);
+
+static const rb_data_type_t graph_typed_data = {
+  "PageRank/SparseNative/Graph",
+  { 0, free_graph, },
+  0, 0,
+  RUBY_TYPED_FREE_IMMEDIATELY,
+};
+
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void Init_sparse_native() {
+  VALUE PageRankModule, SparseNativeClass;
+
+  PageRankModule = rb_const_get(rb_cObject, rb_intern("PageRank"));
+  SparseNativeClass = rb_const_get(PageRankModule, rb_intern("SparseNative"));
+
+  rb_define_alloc_func(SparseNativeClass, sparse_native_allocate);
+  rb_define_private_method(SparseNativeClass, "_add_edge", sparse_native_add_edge, 3);
+  rb_define_private_method(SparseNativeClass, "_calculate", sparse_native_calculate, 3);
+}
+
+VALUE sparse_native_allocate(VALUE self) {
+  Graph g = malloc(GRAPH_SIZE);
+
+  // Grab a reference to the hash type used by a generic Ruby {}
+  // which accepts any key and any value.  We'll need this type to create
+  // a st_table in which to put arbitrary VALUE keys.  This hash type
+  // should be a static constant and thus should be safe to utilize without
+  // fear of garbage collection.
+  const struct st_hash_type *objhash = rb_hash_tbl(rb_hash_new(), "page_rank_sparse_native.c", 40)->type;
+
+  g->node_count = 0;
+  g->nodes = NULL;
+  g->dangling_nodes = NULL;
+  g->node_lookup = st_init_table_with_size(objhash, 0);
+
+  return TypedData_Wrap_Struct(self, &graph_typed_data, g);
+}
+
+VALUE sparse_native_add_edge(VALUE self, VALUE source, VALUE dest, VALUE weight) {
+  Graph g;
+
+  TypedData_Get_Struct(self, GraphStruct, &graph_typed_data, g);
+  add_edge_with_labels(g, source, dest, NUM2DBL(weight));
+  return Qnil;
+}
+
+VALUE sparse_native_calculate(VALUE self, VALUE max_iterations, VALUE damping, VALUE tolerance) {
+  Graph g;
+  VALUE ranks;
+
+  TypedData_Get_Struct(self, GraphStruct, &graph_typed_data, g);
+  calculate(g, FIX2INT(max_iterations), NUM2DBL(damping), NUM2DBL(tolerance));
+
+  ranks = rb_hash_new();
+  sort_and_normalize_ranks(g, rb_hash_dset, ranks);
+  return ranks;
+}
+
+void rb_hash_dset(VALUE hash, VALUE key, double value) {
+  rb_hash_aset(hash, key, DBL2NUM(value));
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void free_graph(void *data) {
+  Graph g = (Graph)data;
+  free_node_list(g->nodes, free_node);
+  free_node_list(g->dangling_nodes, NULL);
+  free(g->node_lookup);
+  free(g);
+}
+
+void free_node(Node n) {
+  free_edge_list(n->source_edges, free_edge);
+  free(n);
+}
+
+void free_node_list(NodeList nodes, void (*free_item)(Node)) {
+  NodeList tmp;
+  while (nodes != NULL) {
+    tmp = nodes;
+    nodes = nodes->next;
+    if (free_item) {
+      free_item(tmp->node);
+    }
+    free(tmp);
+  }
+}
+
+void free_edge(Edge e) {
+  // Assume source node was allocated elsewhere and will be free'd elsewhere
+  free(e);
+}
+
+void free_edge_list(EdgeList edges, void (*free_item)(Edge)) {
+  EdgeList tmp;
+  while (edges != NULL) {
+    tmp = edges;
+    edges = edges->next;
+    if (free_item) {
+      free_item(tmp->edge);
+    }
+    free(tmp);
+  }
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+Node add_node(Graph g, VALUE label) {
+  NodeList tmp = malloc(NODE_LIST_SIZE);
+
+  tmp->node = malloc(NODE_SIZE);
+  tmp->node->label = label;
+  tmp->node->source_edges = NULL;
+  tmp->node->rank = 0.0;
+  tmp->node->prev_rank = 0.0;
+  tmp->node->outbound_weight_total = 0.0;
+
+  tmp->next = g->nodes;
+  g->nodes = tmp;
+  g->node_count += 1;
+
+  return tmp->node;
+}
+
+Node add_dangling_node(Graph g, Node n) {
+  NodeList tmp = malloc(NODE_LIST_SIZE);
+
+  tmp->node = n;
+  tmp->next = g->dangling_nodes;
+  g->dangling_nodes = tmp;
+
+  return n;
+}
+
+Edge add_edge(Node source, Node destination, double weight) {
+  EdgeList tmp = malloc(EDGE_LIST_SIZE);
+
+  tmp->edge = malloc(EDGE_SIZE);
+  tmp->edge->source = source;
+  tmp->edge->weight = weight;
+
+  tmp->next = destination->source_edges;
+  destination->source_edges = tmp;
+  source->outbound_weight_total += weight;
+
+  return tmp->edge;
+}
+
+Edge add_edge_with_labels(Graph g, VALUE source_label, VALUE dest_label, double weight) {
+  Node source, dest;
+
+  source = lookup_node(g, source_label);
+  dest = lookup_node(g, dest_label);
+
+  return add_edge(source, dest, weight);
+}
+
+Node lookup_node(Graph g, VALUE label) {
+  Node n;
+
+  if (!st_lookup(g->node_lookup, (st_data_t)label, (st_data_t *)&n)) {
+    n = add_node(g, label);
+    st_add_direct(g->node_lookup, (st_data_t)label, (st_data_t)n);
+  }
+  return n;
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void calculate_start(Graph g) {
+  NodeList nodes;
+  Node source, destination;
+  EdgeList edges;
+  Edge e;
+
+  for (nodes = g->nodes; nodes != NULL; nodes = nodes->next) {
+    destination = nodes->node;
+
+    // If there is no outband, this is a "dangling" node
+    if (destination->outbound_weight_total == 0.0) {
+      add_dangling_node(g, destination);
+    }
+
+    // Normalize all source edge weights
+    for (edges = destination->source_edges; edges != NULL; edges = edges->next) {
+      e = edges->edge;
+      source = e->source;
+      e->weight = e->weight / source->outbound_weight_total;
+    }
+
+    // Set the initial rank
+    destination->prev_rank = 0;
+    destination->rank = 1.0 / g->node_count;
+  }
+}
+
+void calculate_step(Graph g, double damping) {
+  NodeList nodes, dangling_nodes;
+  Node source, destination;
+  EdgeList edges;
+  Edge e;
+  double sum;
+
+  // Set prev rank to rank for all nodes
+  for (nodes = g->nodes; nodes != NULL; nodes = nodes->next) {
+    destination = nodes->node;
+    destination->prev_rank = destination->rank;
+  }
+
+  // Re-destribute the rankings according to weight
+  for (nodes = g->nodes; nodes != NULL; nodes = nodes->next) {
+    destination = nodes->node;
+    sum = 0.0;
+    for (edges = destination->source_edges; edges != NULL; edges = edges->next) {
+      e = edges->edge;
+      source = e->source;
+      sum += source->prev_rank * e->weight;
+    }
+    for (dangling_nodes = g->dangling_nodes; dangling_nodes != NULL; dangling_nodes = dangling_nodes->next) {
+      source = dangling_nodes->node;
+      sum += source->prev_rank / g->node_count;
+    }
+    destination->rank = damping * sum + (1 - damping) / g->node_count;
+  }
+}
+
+// Calculate the Euclidean distance from prev_rank to rank across all nodes
+double prev_distance(Graph g) {
+  NodeList nodes;
+  Node n;
+  double rank_diff, sum_squares = 0.0;
+
+  for (nodes = g->nodes; nodes != NULL; nodes = nodes->next) {
+    n = nodes->node;
+    rank_diff = n->prev_rank - n->rank;
+    sum_squares += rank_diff * rank_diff;
+  }
+
+  return sqrt(sum_squares);
+}
+
+void calculate(Graph g, int max_iterations, double damping, double tolerance) {
+  calculate_start(g);
+
+  while (max_iterations != 0) { // If negative one, allow to go without limit
+    calculate_step(g, damping);
+    if (prev_distance(g) < tolerance) {
+      break;
+    }
+    max_iterations--;
+  }
+}
+
+int node_compare(const void *v1, const void *v2) {
+  double rank1, rank2, cmp;
+
+  rank1 = (*(Node *)v1)->rank;
+  rank2 = (*(Node *)v2)->rank;
+  cmp = rank2 - rank1; // Decreasing order
+  if (cmp < 0) return -1;
+  if (cmp > 0) return 1;
+  return 0;
+}
+
+void sort_and_normalize_ranks(Graph g, void (*callback)(VALUE, VALUE, double), VALUE callback_arg) {
+  NodeList nodes;
+  Node n;
+  double sum = 0.0;
+  unsigned long i;
+  Node *tmp;
+
+  i = g->node_count;
+  tmp = malloc(g->node_count * sizeof(Node));
+  for (nodes = g->nodes; nodes != NULL; nodes = nodes->next) {
+    n = nodes->node;
+    tmp[--i] = n;
+    sum += n->rank;
+  }
+
+  qsort(tmp, g->node_count, sizeof(Node), node_compare);
+
+  for (i = 0; i < g->node_count; i++) {
+    n = tmp[i];
+    callback(callback_arg, n->label, n->rank / sum);
+  }
+
+  free(tmp);
+}

data/ext/text_rank/page_rank_sparse_native.h

@@ -0,0 +1,93 @@
+#ifndef PAGE_RANK_SPARSE_NATIVE_H
+#define PAGE_RANK_SPARSE_NATIVE_H
+
+#include <ruby.h>
+
+struct NodeListStruct;
+typedef struct NodeListStruct* NodeList;
+
+typedef struct NodeListStruct {
+  struct NodeStruct *node;
+  struct NodeListStruct *next;
+} NodeListStruct;
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+struct EdgeListStruct;
+typedef struct EdgeListStruct* EdgeList;
+
+typedef struct EdgeListStruct {
+  struct EdgeStruct *edge;
+  struct EdgeListStruct *next;
+} EdgeListStruct;
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+struct NodeStruct;
+typedef struct NodeStruct* Node;
+
+typedef struct NodeStruct {
+  EdgeList source_edges;
+  VALUE label;
+  double prev_rank;
+  double rank;
+  double outbound_weight_total;
+} NodeStruct;
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+struct EdgeStruct;
+typedef struct EdgeStruct* Edge;
+
+typedef struct EdgeStruct {
+  Node source;
+  double weight;
+} EdgeStruct;
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+struct GraphStruct;
+typedef struct GraphStruct* Graph;
+
+typedef struct GraphStruct {
+  unsigned long node_count;
+  NodeList nodes;
+  NodeList dangling_nodes;
+  st_table *node_lookup;
+} GraphStruct;
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void free_graph(void *data);
+void free_node(Node n);
+void free_node_list(NodeList nodes, void (*free_item)(Node));
+void free_edge(Edge e);
+void free_edge_list(EdgeList edges, void (*free_item)(Edge));
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+Node add_node(Graph g, VALUE label);
+Node add_dangling_node(Graph g, Node n);
+Edge add_edge(Node source, Node destination, double weight);
+Edge add_edge_with_labels(Graph g, VALUE source_label, VALUE dest_label, double weight);
+Node lookup_node(Graph g, VALUE label);
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void calculate_start(Graph g);
+void calculate_step(Graph g, double damping);
+double prev_distance(Graph g);
+void calculate(Graph g, int max_iterations, double damping, double tolerance);
+int node_compare(const void *v1, const void *v2);
+void sort_and_normalize_ranks(Graph g, void (*callback)(VALUE, VALUE, double), VALUE callback_arg);
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void Init_sparse_native();
+VALUE sparse_native_allocate(VALUE self);
+VALUE sparse_native_add_edge(VALUE self, VALUE source, VALUE dest, VALUE weight);
+VALUE sparse_native_calculate(VALUE self, VALUE max_iterations, VALUE damping, VALUE tolerance);
+VALUE sorted_and_normalized_ranks(Graph g);
+void rb_hash_dset(VALUE hash, VALUE key, double value);
+
+#endif

data/ext/text_rank/text_rank.c

@@ -0,0 +1,5 @@
+#include <page_rank_sparse_native.h>
+
+void Init_text_rank() {
+  Init_sparse_native();
+}

data/lib/page_rank/base.rb

@@ -7,6 +7,8 @@ module PageRank
   ##
   class Base
 
+    attr_reader :damping, :tolerance
+
     # @param (see #damping=)
     # @param (see #tolerance=)
     def initialize(damping: nil, tolerance: nil, **_)
@@ -19,8 +21,7 @@ module PageRank
     # @return [Float]
     def damping=(damping)
       @damping = damping || 0.85
-      raise ArgumentError.new('Invalid damping factor') if @damping <= 0 || @damping > 1
-      @damping
+      raise ArgumentError, 'Invalid damping factor' if @damping <= 0 || @damping > 1
     end
 
     # Set the tolerance value
@@ -28,8 +29,7 @@ module PageRank
     # @return [Float]
     def tolerance=(tolerance)
       @tolerance = tolerance || 0.0001
-      raise ArgumentError.new('Invalid tolerance factor') if @tolerance < 0 || @tolerance > 1
-      @tolerance
+      raise ArgumentError, 'Invalid tolerance factor' if @tolerance.negative? || @tolerance > 1
     end
 
     # Adds a directed (and optionally weighted) edge to the graph
@@ -46,9 +46,12 @@ module PageRank
     def calculate(max_iterations: -1, **_)
       ranks = initial_ranks
       loop do
-        break if max_iterations == 0
-        ranks, prev_ranks = calculate_step(ranks), ranks
-        break if distance(ranks, prev_ranks) < @tolerance
+        break if max_iterations.zero?
+
+        prev_ranks = ranks
+        ranks = calculate_step(ranks)
+        break if distance(ranks, prev_ranks) < tolerance
+
         max_iterations -= 1
       end
       sort_ranks(ranks)
@@ -77,9 +80,9 @@ module PageRank
     end
 
     # Calculate the Euclidean distance from one ranking to the next iteration
-    def distance(v1, v2)
+    def distance(vector1, vector2)
       sum_squares = node_count.times.reduce(0.0) do |sum, i|
-        d = v1[i] - v2[i]
+        d = vector1[i] - vector2[i]
         sum + d * d
       end
       Math.sqrt(sum_squares)

data/lib/page_rank/dense.rb

@@ -32,6 +32,7 @@ module PageRank
     # @return (see Base#add)
     def add(source, dest, weight: 1.0)
       return if source == dest
+
       source_idx = index(source)
       dest_idx = index(dest)
       @out_links[source_idx] ||= []
@@ -72,13 +73,13 @@ module PageRank
 
     def to_matrix
       total_out_weights = @out_links.map do |links|
-        links.compact.reduce(:+) if links
+        links&.compact&.reduce(:+)
       end
       Matrix.build(node_count, node_count) do |dest_idx, source_idx|
         total = total_out_weights[source_idx]
         if total
           w = @out_links[source_idx][dest_idx] || 0.0
-          @damping * w / total + (1 - @damping) / node_count.to_f
+          damping * w / total + (1 - damping) / node_count.to_f
         else
           1.0 / node_count.to_f
         end

data/lib/page_rank/sparse.rb

@@ -1,5 +1,3 @@
-require 'set'
-
 module PageRank
   ##
   # Implementation of PageRank using a sparse matrix representation of the graph
@@ -33,6 +31,7 @@ module PageRank
     # @return (see Base#add)
     def add(source, dest, weight: 1.0)
       return false if source == dest
+
       @graph[dest] ||= Set.new
       @graph[dest] << source
       @weights[source] ||= Hash.new(0.0)
@@ -53,8 +52,8 @@ module PageRank
     def initial_ranks
       @dangling_nodes = @nodes - @weight_totals.keys
       @normalized_weights = @weights.each_with_object({}) do |(source, values), h|
-        h[source] = values.each_with_object({}) do |(dest, w), h2| 
-          h2[dest] = w / @weight_totals[source]
+        h[source] = values.transform_values do |w|
+          w / @weight_totals[source]
         end
       end
       Hash[@nodes.map { |k| [k, 1.0 / node_count.to_f] }]
@@ -69,7 +68,7 @@ module PageRank
         @dangling_nodes.each do |source|
           sum += ranks[source] / node_count.to_f
         end
-        new_ranks[dest] = @damping * sum + (1 - @damping)/node_count
+        new_ranks[dest] = damping * sum + (1 - damping) / node_count
       end
     end
 
@@ -79,8 +78,8 @@ module PageRank
       Hash[ranks.map { |k, v| [k, v / sum] }.sort_by { |_, v| -v }]
     end
 
-    def distance(v1, v2)
-      super(v1.values.to_a, v2.values.to_a)
+    def distance(vector1, vector2)
+      super(vector1.values.to_a, vector2.values.to_a)
     end
 
   end

data/lib/page_rank/sparse_native.rb

@@ -0,0 +1,21 @@
+module PageRank
+  class SparseNative < Base
+
+    #require 'page_rank/sparse_native.so'
+
+    # @param (see Base#add)
+    # @param weight [Float] Optional weight for the graph edge
+    # @return (see Base#add)
+    def add(source, dest, weight: 1.0)
+      _add_edge(source, dest, weight) unless source == dest
+    end
+
+    # Perform the PageRank calculation
+    # @param max_iterations [Fixnum] Maximum number of PageRank iterations to perform (or -1 for no max)
+    # @return [Hash<Object, Float>] of nodes with rank
+    def calculate(max_iterations: -1, **_)
+      _calculate(max_iterations, damping, tolerance)
+    end
+
+  end
+end

data/lib/page_rank.rb

@@ -1,3 +1,5 @@
+require 'set'
+
 ##
 # A module for supporting Ruby implementations of PageRank. Rather than rely on
 # one single implementation, this module allows for multiple implementations that
@@ -15,16 +17,17 @@
 ##
 module PageRank
 
-  autoload :Base,   'page_rank/base'
-  autoload :Dense,  'page_rank/dense'
-  autoload :Sparse, 'page_rank/sparse'
+  autoload :Base,         'page_rank/base'
+  autoload :Dense,        'page_rank/dense'
+  autoload :Sparse,       'page_rank/sparse'
+  autoload :SparseNative, 'page_rank/sparse_native'
 
   # @option options [Symbol] :strategy PageRank strategy to use (either :sparse or :dense)
   # @option options [Float]  :damping The probability of following the graph vs. randomly choosing a new node
   # @option options [Float]  :tolerance The desired accuracy of the results
   # @return [PageRank::Base]
   def self.new(strategy: :sparse, **options)
-    const_get(strategy.to_s.capitalize).new(**options)
+    const_get(strategy.to_s.split('_').map(&:capitalize).join).new(**options)
   end
 
   # Convenience method to quickly calculate PageRank. In the calling block, graph edges can be added.