text_rank 1.2.5 → 1.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8578660270e5eb59c1bd9cfc7b3a22d10ac953cc6672073b368bbf20464dc3f3
-  data.tar.gz: 6043c2eea693623b65f2dffc0ad2be1ae5bb0f1be57dcbbcb8ad871c4260a63b
+  metadata.gz: ea53630f7c00d0731b2a190c8c8775b7f433557611b85b17eae8c0989bfac108
+  data.tar.gz: 184389405a3ddbf216290f2a9bc806c5c70484dea4b677a89f089bfb0054f161
 SHA512:
-  metadata.gz: 73acab105c3f6dbee7b7c49bd383f41b0da192d1c67b56c9813f7ad1f32c70790b7a99f334bccd3104573e78f39cd5400f0ea066bf2f776da7f6c7083b4d9df2
-  data.tar.gz: c51e2861b1aaa21ce91c783d3528815b66c343d8299564a50e69ad0887b14d06908d54626fbd01ed8508f52cd8d893433d3c91f5eec6f8a6feff4bbc077c8a17
+  metadata.gz: 4cce8fa0323a59e52bb1b0f220f93c8dfdb68b795f7027e1f871c563d331083dbe4803b3176b14f15d2de5dc7c9d533af8104e125149ff05af397b88f02df6bb
+  data.tar.gz: 07efa700bbc61c56ca440a0d241a576a531f4caa4f26c48b7e8a938f19cdadf5d59d1d6bcf46d3cfc43ca4c5a4ada865ef2dbb4e6d376abefed283c6a410c719

data/.gitignore

@@ -8,3 +8,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+
+
+*.bundle
+*.so

data/.travis.yml

@@ -9,6 +9,7 @@ before_script:
   - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
   - chmod +x ./cc-test-reporter
   - ./cc-test-reporter before-build
+  - bundle exec rake compile
 script:
   - bundle exec rspec
 after_script:

data/Rakefile

@@ -1,4 +1,5 @@
 require "bundler/gem_tasks"
+require "rake/extensiontask"
 require "rspec/core/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
@@ -10,3 +11,7 @@ RDoc::Task.new do |rdoc|
   rdoc.main = "README.md"
   rdoc.rdoc_files.include("README.md", "lib/**/*.rb")
 end
+
+Rake::ExtensionTask.new('text_rank') do |ext|
+  ext.lib_dir = 'lib/text_rank'
+end

data/ext/text_rank/extconf.rb

@@ -0,0 +1,3 @@
+require "mkmf"
+
+create_makefile('text_rank/text_rank')

data/ext/text_rank/page_rank_sparse_native.c

@@ -0,0 +1,296 @@
+#include <ruby.h>
+#include <math.h>
+#include <page_rank_sparse_native.h>
+
+const size_t NODE_LIST_SIZE = sizeof(NodeListStruct);
+const size_t EDGE_LIST_SIZE = sizeof(EdgeListStruct);
+const size_t NODE_SIZE = sizeof(NodeStruct);
+const size_t EDGE_SIZE = sizeof(EdgeStruct);
+const size_t GRAPH_SIZE = sizeof(GraphStruct);
+
+static const rb_data_type_t graph_typed_data = {
+  "PageRank/SparseNative/Graph",
+  { 0, free_graph, },
+  0, 0,
+  RUBY_TYPED_FREE_IMMEDIATELY,
+};
+
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void Init_sparse_native() {
+  VALUE PageRankModule, SparseNativeClass;
+
+  PageRankModule = rb_const_get(rb_cObject, rb_intern("PageRank"));
+  SparseNativeClass = rb_const_get(PageRankModule, rb_intern("SparseNative"));
+
+  rb_define_alloc_func(SparseNativeClass, sparse_native_allocate);
+  rb_define_private_method(SparseNativeClass, "_add_edge", sparse_native_add_edge, 3);
+  rb_define_private_method(SparseNativeClass, "_calculate", sparse_native_calculate, 3);
+}
+
+VALUE sparse_native_allocate(VALUE self) {
+  Graph g = malloc(GRAPH_SIZE);
+  //st_table *tmp, *node_lookup;
+
+  const struct st_hash_type *objhash = rb_hash_tbl(rb_hash_new())->type;
+
+  g->node_count = 0;
+  g->nodes = NULL;
+  g->dangling_nodes = NULL;
+  g->node_lookup = st_init_table_with_size(objhash, 0);
+
+  return TypedData_Wrap_Struct(self, &graph_typed_data, g);
+}
+
+VALUE sparse_native_add_edge(VALUE self, VALUE source, VALUE dest, VALUE weight) {
+  Graph g;
+
+  TypedData_Get_Struct(self, GraphStruct, &graph_typed_data, g);
+  add_edge_with_labels(g, source, dest, NUM2DBL(weight));
+  return Qnil;
+}
+
+VALUE sparse_native_calculate(VALUE self, VALUE max_iterations, VALUE damping, VALUE tolerance) {
+  Graph g;
+  VALUE ranks;
+
+  TypedData_Get_Struct(self, GraphStruct, &graph_typed_data, g);
+  calculate(g, FIX2INT(max_iterations), NUM2DBL(damping), NUM2DBL(tolerance));
+
+  ranks = rb_hash_new();
+  sort_and_normalize_ranks(g, rb_hash_dset, ranks);
+  return ranks;
+}
+
+void rb_hash_dset(VALUE hash, VALUE key, double value) {
+  rb_hash_aset(hash, key, DBL2NUM(value));
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void free_graph(void *data) {
+  Graph g = (Graph)data;
+  free_node_list(g->nodes, free_node);
+  free_node_list(g->dangling_nodes, NULL);
+  free(g->node_lookup);
+  free(g);
+}
+
+void free_node(Node n) {
+  free_edge_list(n->source_edges, free_edge);
+  free(n);
+}
+
+void free_node_list(NodeList nodes, void (*free_item)(Node)) {
+  NodeList tmp;
+  while (nodes != NULL) {
+    tmp = nodes;
+    nodes = nodes->next;
+    if (free_item) {
+      free_item(tmp->node);
+    }
+    free(tmp);
+  }
+}
+
+void free_edge(Edge e) {
+  // Assume source node was allocated elsewhere and will be free'd elsewhere
+  free(e);
+}
+
+void free_edge_list(EdgeList edges, void (*free_item)(Edge)) {
+  EdgeList tmp;
+  while (edges != NULL) {
+    tmp = edges;
+    edges = edges->next;
+    if (free_item) {
+      free_item(tmp->edge);
+    }
+    free(tmp);
+  }
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+Node add_node(Graph g, VALUE label) {
+  NodeList tmp = malloc(NODE_LIST_SIZE);
+
+  tmp->node = malloc(NODE_SIZE);
+  tmp->node->label = label;
+  tmp->node->source_edges = NULL;
+  tmp->node->rank = 0.0;
+  tmp->node->prev_rank = 0.0;
+  tmp->node->outbound_weight_total = 0.0;
+
+  tmp->next = g->nodes;
+  g->nodes = tmp;
+  g->node_count += 1;
+
+  return tmp->node;
+}
+
+Node add_dangling_node(Graph g, Node n) {
+  NodeList tmp = malloc(NODE_LIST_SIZE);
+
+  tmp->node = n;
+  tmp->next = g->dangling_nodes;
+  g->dangling_nodes = tmp;
+
+  return n;
+}
+
+Edge add_edge(Node source, Node destination, double weight) {
+  EdgeList tmp = malloc(EDGE_LIST_SIZE);
+
+  tmp->edge = malloc(EDGE_SIZE);
+  tmp->edge->source = source;
+  tmp->edge->weight = weight;
+
+  tmp->next = destination->source_edges;
+  destination->source_edges = tmp;
+  source->outbound_weight_total += weight;
+
+  return tmp->edge;
+}
+
+Edge add_edge_with_labels(Graph g, VALUE source_label, VALUE dest_label, double weight) {
+  Node source, dest;
+
+  source = lookup_node(g, source_label);
+  dest = lookup_node(g, dest_label);
+
+  return add_edge(source, dest, weight);
+}
+
+Node lookup_node(Graph g, VALUE label) {
+  Node n;
+
+  if (!st_lookup(g->node_lookup, (st_data_t)label, (st_data_t *)&n)) {
+    n = add_node(g, label);
+    st_add_direct(g->node_lookup, (st_data_t)label, (st_data_t)n);
+  }
+  return n;
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void calculate_start(Graph g) {
+  NodeList nodes;
+  Node source, destination;
+  EdgeList edges;
+  Edge e;
+
+  for (nodes = g->nodes; nodes != NULL; nodes = nodes->next) {
+    destination = nodes->node;
+
+    // If there is no outband, this is a "dangling" node
+    if (destination->outbound_weight_total == 0.0) {
+      add_dangling_node(g, destination);
+    }
+
+    // Normalize all source edge weights
+    for (edges = destination->source_edges; edges != NULL; edges = edges->next) {
+      e = edges->edge;
+      source = e->source;
+      e->weight = e->weight / source->outbound_weight_total;
+    }
+
+    // Set the initial rank
+    destination->prev_rank = 0;
+    destination->rank = 1.0 / g->node_count;
+  }
+}
+
+void calculate_step(Graph g, double damping) {
+  NodeList nodes, dangling_nodes;
+  Node source, destination;
+  EdgeList edges;
+  Edge e;
+  double sum;
+
+  // Set prev rank to rank for all nodes
+  for (nodes = g->nodes; nodes != NULL; nodes = nodes->next) {
+    destination = nodes->node;
+    destination->prev_rank = destination->rank;
+  }
+
+  // Re-destribute the rankings according to weight
+  for (nodes = g->nodes; nodes != NULL; nodes = nodes->next) {
+    destination = nodes->node;
+    sum = 0.0;
+    for (edges = destination->source_edges; edges != NULL; edges = edges->next) {
+      e = edges->edge;
+      source = e->source;
+      sum += source->prev_rank * e->weight;
+    }
+    for (dangling_nodes = g->dangling_nodes; dangling_nodes != NULL; dangling_nodes = dangling_nodes->next) {
+      source = dangling_nodes->node;
+      sum += source->prev_rank / g->node_count;
+    }
+    destination->rank = damping * sum + (1 - damping) / g->node_count;
+  }
+}
+
+// Calculate the Euclidean distance from prev_rank to rank across all nodes
+double prev_distance(Graph g) {
+  NodeList nodes;
+  Node n;
+  double rank_diff, sum_squares = 0.0;
+
+  for (nodes = g->nodes; nodes != NULL; nodes = nodes->next) {
+    n = nodes->node;
+    rank_diff = n->prev_rank - n->rank;
+    sum_squares += rank_diff * rank_diff;
+  }
+
+  return sqrt(sum_squares);
+}
+
+void calculate(Graph g, int max_iterations, double damping, double tolerance) {
+  calculate_start(g);
+
+  while (max_iterations != 0) { // If negative one, allow to go without limit
+    calculate_step(g, damping);
+    if (prev_distance(g) < tolerance) {
+      break;
+    }
+    max_iterations--;
+  }
+}
+
+int node_compare(const void *v1, const void *v2) {
+  double rank1, rank2, cmp;
+
+  rank1 = (*(Node *)v1)->rank;
+  rank2 = (*(Node *)v2)->rank;
+  cmp = rank2 - rank1; // Decreasing order
+  if (cmp < 0) return -1;
+  if (cmp > 0) return 1;
+  return 0;
+}
+
+void sort_and_normalize_ranks(Graph g, void (*callback)(VALUE, VALUE, double), VALUE callback_arg) {
+  NodeList nodes;
+  Node n;
+  double sum = 0.0;
+  unsigned long i;
+  Node *tmp;
+
+  i = g->node_count;
+  tmp = malloc(g->node_count * sizeof(Node));
+  for (nodes = g->nodes; nodes != NULL; nodes = nodes->next) {
+    n = nodes->node;
+    tmp[--i] = n;
+    sum += n->rank;
+  }
+
+  qsort(tmp, g->node_count, sizeof(Node), node_compare);
+
+  for (i = 0; i < g->node_count; i++) {
+    n = tmp[i];
+    callback(callback_arg, n->label, n->rank / sum);
+  }
+
+  free(tmp);
+}

data/ext/text_rank/page_rank_sparse_native.h

@@ -0,0 +1,93 @@
+#ifndef PAGE_RANK_SPARSE_NATIVE_H
+#define PAGE_RANK_SPARSE_NATIVE_H
+
+#include <ruby.h>
+
+struct NodeListStruct;
+typedef struct NodeListStruct* NodeList;
+
+typedef struct NodeListStruct {
+  struct NodeStruct *node;
+  struct NodeListStruct *next;
+} NodeListStruct;
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+struct EdgeListStruct;
+typedef struct EdgeListStruct* EdgeList;
+
+typedef struct EdgeListStruct {
+  struct EdgeStruct *edge;
+  struct EdgeListStruct *next;
+} EdgeListStruct;
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+struct NodeStruct;
+typedef struct NodeStruct* Node;
+
+typedef struct NodeStruct {
+  EdgeList source_edges;
+  VALUE label;
+  double prev_rank;
+  double rank;
+  double outbound_weight_total;
+} NodeStruct;
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+struct EdgeStruct;
+typedef struct EdgeStruct* Edge;
+
+typedef struct EdgeStruct {
+  Node source;
+  double weight;
+} EdgeStruct;
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+struct GraphStruct;
+typedef struct GraphStruct* Graph;
+
+typedef struct GraphStruct {
+  unsigned long node_count;
+  NodeList nodes;
+  NodeList dangling_nodes;
+  st_table *node_lookup;
+} GraphStruct;
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void free_graph(void *data);
+void free_node(Node n);
+void free_node_list(NodeList nodes, void (*free_item)(Node));
+void free_edge(Edge e);
+void free_edge_list(EdgeList edges, void (*free_item)(Edge));
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+Node add_node(Graph g, VALUE label);
+Node add_dangling_node(Graph g, Node n);
+Edge add_edge(Node source, Node destination, double weight);
+Edge add_edge_with_labels(Graph g, VALUE source_label, VALUE dest_label, double weight);
+Node lookup_node(Graph g, VALUE label);
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void calculate_start(Graph g);
+void calculate_step(Graph g, double damping);
+double prev_distance(Graph g);
+void calculate(Graph g, int max_iterations, double damping, double tolerance);
+int node_compare(const void *v1, const void *v2);
+void sort_and_normalize_ranks(Graph g, void (*callback)(VALUE, VALUE, double), VALUE callback_arg);
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+void Init_sparse_native();
+VALUE sparse_native_allocate(VALUE self);
+VALUE sparse_native_add_edge(VALUE self, VALUE source, VALUE dest, VALUE weight);
+VALUE sparse_native_calculate(VALUE self, VALUE max_iterations, VALUE damping, VALUE tolerance);
+VALUE sorted_and_normalized_ranks(Graph g);
+void rb_hash_dset(VALUE hash, VALUE key, double value);
+
+#endif

data/ext/text_rank/text_rank.c

@@ -0,0 +1,5 @@
+#include <page_rank_sparse_native.h>
+
+void Init_text_rank() {
+  Init_sparse_native();
+}

data/lib/page_rank.rb

@@ -17,16 +17,17 @@ require 'set'
 ##
 module PageRank
 
-  autoload :Base,   'page_rank/base'
-  autoload :Dense,  'page_rank/dense'
-  autoload :Sparse, 'page_rank/sparse'
+  autoload :Base,         'page_rank/base'
+  autoload :Dense,        'page_rank/dense'
+  autoload :Sparse,       'page_rank/sparse'
+  autoload :SparseNative, 'page_rank/sparse_native'
 
   # @option options [Symbol] :strategy PageRank strategy to use (either :sparse or :dense)
   # @option options [Float]  :damping The probability of following the graph vs. randomly choosing a new node
   # @option options [Float]  :tolerance The desired accuracy of the results
   # @return [PageRank::Base]
   def self.new(strategy: :sparse, **options)
-    const_get(strategy.to_s.capitalize).new(**options)
+    const_get(strategy.to_s.split('_').map(&:capitalize).join).new(**options)
   end
 
   # Convenience method to quickly calculate PageRank. In the calling block, graph edges can be added.

data/lib/page_rank/base.rb

@@ -7,6 +7,8 @@ module PageRank
   ##
   class Base
 
+    attr_reader :damping, :tolerance
+
     # @param (see #damping=)
     # @param (see #tolerance=)
     def initialize(damping: nil, tolerance: nil, **_)
@@ -48,7 +50,7 @@ module PageRank
 
         prev_ranks = ranks
         ranks = calculate_step(ranks)
-        break if distance(ranks, prev_ranks) < @tolerance
+        break if distance(ranks, prev_ranks) < tolerance
 
         max_iterations -= 1
       end

data/lib/page_rank/dense.rb

@@ -79,7 +79,7 @@ module PageRank
         total = total_out_weights[source_idx]
         if total
           w = @out_links[source_idx][dest_idx] || 0.0
-          @damping * w / total + (1 - @damping) / node_count.to_f
+          damping * w / total + (1 - damping) / node_count.to_f
         else
           1.0 / node_count.to_f
         end

data/lib/page_rank/sparse.rb

@@ -68,7 +68,7 @@ module PageRank
         @dangling_nodes.each do |source|
           sum += ranks[source] / node_count.to_f
         end
-        new_ranks[dest] = @damping * sum + (1 - @damping) / node_count
+        new_ranks[dest] = damping * sum + (1 - damping) / node_count
       end
     end
 

data/lib/page_rank/sparse_native.rb

@@ -0,0 +1,21 @@
+module PageRank
+  class SparseNative < Base
+
+    #require 'page_rank/sparse_native.so'
+
+    # @param (see Base#add)
+    # @param weight [Float] Optional weight for the graph edge
+    # @return (see Base#add)
+    def add(source, dest, weight: 1.0)
+      _add_edge(source, dest, weight) unless source == dest
+    end
+
+    # Perform the PageRank calculation
+    # @param max_iterations [Fixnum] Maximum number of PageRank iterations to perform (or -1 for no max)
+    # @return [Hash<Object, Float>] of nodes with rank
+    def calculate(max_iterations: -1, **_)
+      _calculate(max_iterations, damping, tolerance)
+    end
+
+  end
+end

data/lib/text_rank.rb

@@ -40,3 +40,5 @@ module TextRank
   end
 
 end
+
+require 'text_rank/text_rank'

data/lib/text_rank/version.rb

@@ -1,6 +1,6 @@
 module TextRank
 
   # Current gem version
-  VERSION = '1.2.5'
+  VERSION = '1.2.9'
 
 end

data/text_rank.gemspec

@@ -16,10 +16,12 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.extensions    = ['ext/text_rank/extconf.rb']
   spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rake-compiler'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rubocop'
   spec.add_development_dependency 'simplecov', '~> 0.17.0' # 0.18 not supported by code climate

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_rank
 version: !ruby/object:Gem::Version
-  version: 1.2.5
+  version: 1.2.9
 platform: ruby
 authors:
 - David McCullars
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-27 00:00:00.000000000 Z
+date: 2021-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -127,7 +141,8 @@ description: Implementation of TextRank solution to ranked keyword extraction.
 email:
 - david.mccullars@gmail.com
 executables: []
-extensions: []
+extensions:
+- ext/text_rank/extconf.rb
 extra_rdoc_files: []
 files:
 - ".codeclimate.yml"
@@ -143,10 +158,15 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- ext/text_rank/extconf.rb
+- ext/text_rank/page_rank_sparse_native.c
+- ext/text_rank/page_rank_sparse_native.h
+- ext/text_rank/text_rank.c
 - lib/page_rank.rb
 - lib/page_rank/base.rb
 - lib/page_rank/dense.rb
 - lib/page_rank/sparse.rb
+- lib/page_rank/sparse_native.rb
 - lib/text_rank.rb
 - lib/text_rank/char_filter.rb
 - lib/text_rank/char_filter/ascii_folding.rb