commendo 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 04c833f06518b70ff1370c1685df37294c8fd2b9
-  data.tar.gz: dd99d2a4bcca958d2c216d3084b88298c59f462f
+  metadata.gz: 469912023220e7c8d711e40f37225461644e24db
+  data.tar.gz: f4e79eccf3eee391ff0829720914d82d535d283a
 SHA512:
-  metadata.gz: 9936b22c4e2ffc54e63b36cc13c8dce2d1559115956092c3061ca65e3f5db76302128d5fe46aefb30685394bc1f508a6ca05455bc3e0f2e1ae2ddd6fe9121e3b
-  data.tar.gz: 952208db4416f43325d581f36363a124b0614d656c3b90d38d8b6576350cb8f561aa14022c6084b50b47cc51825a405350682bb6cbb80974590a3110412e0c11
+  metadata.gz: 58a603eb4e13bd0a9f09c3e8e5fb34fd6b86a2a0944c86da3389792b76a505b808a03ec9ab35aa39e0b6152c3ba20dbfb69d83c2dced898b1ae09a152792aa07
+  data.tar.gz: c9af132db981e73748ed917fe42b3aa10f43135bc55f7340570e955c5dbb3c14da0f4d6aa295ae8003f86e923368d9404894fdc545ed8e9e38895718ef636ab1

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# 2.1.0 / 2016-01-04
+* [FEATURE] Commendo can includes a Ruby-backed in-memory implementation (experimental)
+
 # 2.0.0 / 2015-12-11
 * [FEATURE] Commendo can now use MySQL as a backend if you prefer it to Redis
 

data/bin/commendo-time-ruby.rb

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'redis'
+require 'commendo'
+require 'progressbar'
+
+def timer(msg)
+  start = Time.now
+  value = yield
+  finish = Time.now
+  $stderr.puts "#{msg} took #{finish - start}"
+  return value
+end
+
+infile = ARGV[0]
+key_base = ARGV[1]
+limit = ARGV[2].to_i
+
+Commendo.config do |config|
+  config.backend = :ruby
+end
+cs = Commendo::ContentSet.new(key_base: key_base)
+
+resource_to_sets = nil
+timer('Loading') do
+  lines = File.open(infile).readlines
+  resource_to_sets = lines.map { |line| line.strip!; line = line.split("\t"); r = line.shift; [r, line] }
+  resource_to_sets.each do |resource, sets|
+    cs.add(resource, *sets)
+  end
+end
+
+timer('calculate_similarity') do
+  cs.calculate_similarity
+end
+
+names_to_query = timer("Select #{limit} random names to use") do
+  resource_to_sets.map { |resource, sets| resource }.sort_by { rand }.first(limit)
+end
+
+# pbar = ProgressBar.new('Querying similar_to', names_to_query.length)
+names_to_query.each do |name|
+  timer('Similarity') { cs.similar_to(name) }
+  # pbar.inc
+end
+# pbar.finish
+

data/lib/commendo.rb

@@ -17,6 +17,10 @@ require 'commendo/mysql-backed/content_set'
 require 'commendo/mysql-backed/tag_set'
 require 'commendo/mysql-backed/weighted_group'
 
+require 'commendo/ruby-backed/content_set'
+require 'commendo/ruby-backed/tag_set'
+require 'commendo/ruby-backed/weighted_group'
+
 module Commendo
 
   def self.config

data/lib/commendo/content_set.rb

@@ -14,6 +14,7 @@ module Commendo
     def initialize(args)
       @backend = RedisBacked::ContentSet.new(args[:key_base], args[:tag_set]) if Commendo.config.backend == :redis
       @backend = MySqlBacked::ContentSet.new(args[:key_base], args[:tag_set]) if Commendo.config.backend == :mysql
+      @backend = RubyBacked::ContentSet.new(args[:key_base], args[:tag_set]) if Commendo.config.backend == :ruby
       raise 'Unrecognised backend type, try :redis or :mysql' if @backend.nil?
     end
 

data/lib/commendo/ruby-backed/content_set.rb

@@ -0,0 +1,133 @@
+module Commendo
+  module RubyBacked
+
+    class ContentSet
+
+      attr_accessor :key_base, :tag_set
+
+      DEFAULT_LIMIT = 1000
+
+      def initialize(key_base, tag_set = nil)
+        @resource_group_score = Hash.new { |h, k| h[k] = Hash.new { |h, k| h[k] = 0 } }
+        @key_base = key_base
+        @tag_set = tag_set
+        @threshold = nil
+      end
+
+      def add_by_group(group, *resources)
+        resources.map! { |r| r.is_a?(Array) ? r : [r, 1] } #sets default score of 1
+        resources.each { |r| add_single(r[0], group, r[1]) }
+      end
+
+      def add(resource, *groups)
+        resource = resource.to_s
+        groups.map! { |g| g.is_a?(Array) ? g : [g, 1] } #sets default score of 1
+        groups.each do |(group, score)|
+          @resource_group_score[resource][group.to_s] += score
+        end
+      end
+
+      def add_single(resource, group, score)
+        add(resource, [group, score])
+      end
+
+      def add_and_calculate(resource, *groups)
+        add(resource, *groups)
+        calculate_similarity
+      end
+
+      def groups(resource)
+        resource = resource.to_s
+        @resource_group_score[resource].keys
+      end
+
+      def delete(resource)
+        resource = resource.to_s
+        @resource_group_score[resource].each { |group,score| @group_resource_scores[group].delete(resource) }
+        @resource_group_score.delete(resource)
+        @resource_totals.delete(resource)
+      end
+
+      def calculate_similarity_for_resource(resource, threshold = 0)
+        calculate_similarity(threshold)
+      end
+
+      def calculate_similarity(threshold = nil)
+        @resource_totals = Hash[@resource_group_score.map { |resource, groups| [resource, groups.map { |group, score| score }.inject(0, :+)] }]
+        flat_resource_group_score = @resource_group_score.flat_map do |resource, groups|
+          groups.map do |group, score|
+            [resource, group, score]
+          end
+        end
+        @group_resource_scores = Hash.new { |h, k| h[k] = {} }
+        flat_resource_group_score.each { |(resource, group, score)| @group_resource_scores[group][resource] = score }
+
+        @threshold = threshold
+      end
+
+      def similar_to(resource, limit = DEFAULT_LIMIT)
+        if resource.is_a? Array
+          return similar_to_array(resource, limit)
+        else
+          return similar_to_single(resource, limit)
+        end
+      end
+
+      def similar_to_array(resources, limit)
+        similar = resources.flat_map { |r| similar_to_single(r, limit) }
+        similar = similar.group_by { |h| h[:resource] }
+        similar = similar.map { |resource, sims| {resource: resource, similarity: sims.inject(0) { |sum, sim| sum += sim[:similarity] }} }
+        similar.map! { |h| {resource: h[:resource], similarity: h[:similarity].round(3)} }
+        similar.keep_if { |sim| @threshold.nil? || sim[:similarity] > @threshold }
+        similar = similar.sort_by { |h| [h[:similarity], h[:resource]] }.reverse.first(limit)
+        similar
+      end
+
+      def similar_to_single(resource, limit)
+        resource = resource.to_s
+        my_groups = @resource_group_score[resource]
+
+        similar = Hash.new { |h, k| h[k] = 0 }
+
+        my_groups.each do |group, my_score|
+          @group_resource_scores[group].each do |other_resource, score|
+            next if other_resource == resource
+            similarity = (my_score + score).to_f / (@resource_totals[resource] + @resource_totals[other_resource]).to_f
+            similar[other_resource] += similarity
+          end
+        end
+
+        similar = similar.map { |resource, similarity| {resource: resource, similarity: similarity.round(3)} }.sort_by { |h| [h[:similarity], h[:resource]] }.reverse.first(limit)
+        similar.keep_if { |sim| @threshold.nil? || sim[:similarity] > @threshold }
+        similar
+      end
+
+      def filtered_similar_to(resource, options = {})
+        if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?) || @tag_set.empty?
+          return similar_to(resource, options[:limit] || DEFAULT_LIMIT)
+        else
+          similar = similar_to(resource)
+          limit = options[:limit] || similar.length
+          filtered = []
+          similar.each do |s|
+            return filtered if filtered.length >= limit
+            filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
+          end
+          return filtered
+        end
+      end
+
+      def remove_from_groups(resource, *groups)
+        resource = resource.to_s
+        groups.each { |g| @resource_group_score[resource].delete(g.to_s) }
+      end
+
+      def remove_from_groups_and_calculate(resource, *groups)
+        remove_from_groups(resource, *groups)
+        calculate_similarity_for_resource(resource)
+      end
+
+    end
+
+  end
+end

data/lib/commendo/ruby-backed/tag_set.rb

@@ -0,0 +1,43 @@
+module Commendo
+  module RubyBacked
+    class TagSet
+
+      attr_accessor :key_base
+
+      def initialize(key_base)
+        @key_base = key_base
+        @resource_to_tags = Hash.new { |h, k| h[k] = [] }
+      end
+
+      def empty?
+        @resource_to_tags.keys.empty?
+      end
+
+      def get(resource)
+        @resource_to_tags[resource.to_s]
+      end
+
+      def add(resource, *tags)
+        @resource_to_tags[resource.to_s] += tags
+      end
+
+      def set(resource, *tags)
+        @resource_to_tags[resource.to_s] = tags
+      end
+
+      def matches(resource, include, exclude = [])
+        resource_tags = get(resource)
+        can_include = include.nil? || include.empty? || (resource_tags & include).length > 0
+        should_exclude = !exclude.nil? && !exclude.empty? && (resource_tags & exclude).length > 0
+        return can_include && !should_exclude
+      end
+
+      def delete(resource, *tags)
+        @resource_to_tags.delete(resource.to_s) if tags.empty?
+        @resource_to_tags[resource.to_s] -= tags unless tags.empty?
+      end
+
+    end
+  end
+end
+

data/lib/commendo/ruby-backed/weighted_group.rb

@@ -0,0 +1,38 @@
+module Commendo
+  module RubyBacked
+
+    class WeightedGroup
+
+      attr_accessor :content_sets, :key_base, :tag_set
+
+      def initialize(key_base, *content_sets)
+        @key_base = key_base
+        @content_sets = content_sets
+      end
+
+      def similar_to(resource, limit = 0)
+        similar = @content_sets.flat_map { |cs| cs[:cs].similar_to(resource).map { |s| {resource: s[:resource], similarity: (s[:similarity] * cs[:weight]).round(3)} } }
+        grouped = similar.group_by { |r| r[:resource] }
+        totaled_similar = grouped.map { |resource, similar| {resource: resource, similarity: similar.inject(0.0) { |sum, s| sum += s[:similarity] }} }.sort_by { |h| [h[:similarity], h[:resource]] }.reverse
+        limit > 0 ? totaled_similar[0..limit-1] : totaled_similar
+      end
+
+      def filtered_similar_to(resource, options = {})
+        if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?)
+          return similar_to(resource, options[:limit] || 0)
+        else
+          similar = similar_to(resource)
+          limit = options[:limit] || similar.length
+          filtered = []
+          similar.each do |s|
+            return filtered if filtered.length >= limit
+            filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
+          end
+          return filtered
+        end
+      end
+
+    end
+
+  end
+end

data/lib/commendo/tag_set.rb

@@ -7,6 +7,7 @@ module Commendo
     def initialize(args)
       @backend = RedisBacked::TagSet.new(args[:key_base]) if Commendo.config.backend == :redis
       @backend = MySqlBacked::TagSet.new(args[:key_base]) if Commendo.config.backend == :mysql
+      @backend = RubyBacked::TagSet.new(args[:key_base]) if Commendo.config.backend == :ruby
       raise 'Unrecognised backend type, try :redis or :mysql' if @backend.nil?
     end
 

data/lib/commendo/version.rb

@@ -1,3 +1,3 @@
 module Commendo
-  VERSION = '2.0.0'
+  VERSION = '2.1.0'
 end

data/lib/commendo/weighted_group.rb

@@ -6,6 +6,7 @@ module Commendo
     def initialize(args)
       @backend = RedisBacked::WeightedGroup.new(args[:key_base], *args[:content_sets]) if Commendo.config.backend == :redis
       @backend = MySqlBacked::WeightedGroup.new(args[:key_base], *args[:content_sets]) if Commendo.config.backend == :mysql
+      @backend = RubyBacked::WeightedGroup.new(args[:key_base], *args[:content_sets]) if Commendo.config.backend == :ruby
       raise 'Unrecognised backend type, try :redis or :mysql' if @backend.nil?
     end
 

data/test/ruby_content_set_test.rb

@@ -0,0 +1,33 @@
+require_relative 'tests_for_content_sets.rb'
+gem 'minitest'
+require 'minitest/autorun'
+require 'minitest/pride'
+require 'minitest/mock'
+require 'mocha/setup'
+require 'commendo'
+
+module Commendo
+
+  class RubyContentSetTest < Minitest::Test
+
+    def setup
+      Commendo.config do |config|
+        config.backend = :ruby
+      end
+      @key_base = 'CommendoTests'
+      @cs = ContentSet.new(key_base: @key_base)
+    end
+
+    def create_tag_set(kb)
+      Commendo::TagSet.new(key_base: kb)
+    end
+
+    def create_content_set(key_base, ts = nil)
+      Commendo::ContentSet.new(key_base: key_base, tag_set: ts)
+    end
+
+    include TestsForContentSets
+
+  end
+
+end

data/test/ruby_tag_set_test.rb

@@ -0,0 +1,27 @@
+require_relative 'tests_for_tag_sets'
+gem 'minitest'
+require 'minitest/autorun'
+require 'minitest/pride'
+require 'minitest/mock'
+require 'mocha/setup'
+require 'commendo'
+
+module Commendo
+
+  class RubyTagSetTest < Minitest::Test
+
+    def setup
+      Commendo.config do |config|
+        config.backend = :ruby
+      end
+      @ts = TagSet.new(key_base: 'TagSetTest')
+    end
+
+    def create_tag_set(kb)
+      Commendo::TagSet.new(key_base: kb)
+    end
+
+    include TestsForTagSets
+
+  end
+end

data/test/ruby_weighted_group_test.rb

@@ -0,0 +1,43 @@
+require_relative 'tests_for_weighted_groups'
+gem 'minitest'
+require 'minitest/autorun'
+require 'minitest/pride'
+require 'minitest/mock'
+require 'mocha/setup'
+require 'commendo'
+
+module Commendo
+
+  class RubyWeightedGroupTest < Minitest::Test
+
+    def setup
+      @tag_set = TagSet.new(key_base: 'CommendoTests:Tags')
+      @cs1 = ContentSet.new(key_base: 'CommendoTests:ContentSet1', tag_set: @tag_set)
+      @cs2 = ContentSet.new(key_base: 'CommendoTests:ContentSet2', tag_set: @tag_set)
+      @cs3 = ContentSet.new(key_base: 'CommendoTests:ContentSet3', tag_set: @tag_set)
+      (3..23).each do |group|
+        (3..23).each do |res|
+          @cs1.add_by_group(group, res) if res.modulo(group).zero? && res.modulo(2).zero?
+          @cs2.add_by_group(group, res) if res.modulo(group).zero? && res.modulo(3).zero?
+          @cs3.add_by_group(group, res) if res.modulo(group).zero? && res.modulo(6).zero?
+          @tag_set.add(res, 'mod3') if res.modulo(3).zero?
+          @tag_set.add(res, 'mod4') if res.modulo(4).zero?
+          @tag_set.add(res, 'mod5') if res.modulo(5).zero?
+          @tag_set.add(res, 'mod7') if res.modulo(7).zero?
+        end
+      end
+      [@cs1, @cs2, @cs3].each { |cs| cs.calculate_similarity }
+      @weighted_group = Commendo::WeightedGroup.new(key_base: 'CommendoTests:WeightedGroup',
+                                                    content_sets: [{cs: @cs1, weight: 1.0},
+                                                                   {cs: @cs2, weight: 10.0},
+                                                                   {cs: @cs3, weight: 100.0}]
+      )
+    end
+
+    include TestsForWeightedGroups
+
+  end
+
+end
+
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: commendo
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Rob Styles
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-18 00:00:00.000000000 Z
+date: 2016-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -150,6 +150,7 @@ executables:
 - commendo-recommendations-distribution
 - commendo-similarity-distribution
 - commendo-time-mysql.rb
+- commendo-time-ruby.rb
 extensions: []
 extra_rdoc_files: []
 files:
@@ -169,6 +170,7 @@ files:
 - bin/commendo-recommendations-distribution
 - bin/commendo-similarity-distribution
 - bin/commendo-time-mysql.rb
+- bin/commendo-time-ruby.rb
 - commendo.gemspec
 - lib/commendo.rb
 - lib/commendo/configuration.rb
@@ -181,6 +183,9 @@ files:
 - lib/commendo/redis-backed/similarity.lua
 - lib/commendo/redis-backed/tag_set.rb
 - lib/commendo/redis-backed/weighted_group.rb
+- lib/commendo/ruby-backed/content_set.rb
+- lib/commendo/ruby-backed/tag_set.rb
+- lib/commendo/ruby-backed/weighted_group.rb
 - lib/commendo/tag_set.rb
 - lib/commendo/version.rb
 - lib/commendo/weighted_group.rb
@@ -194,6 +199,9 @@ files:
 - test/redis_content_set_test.rb
 - test/redis_tag_set_test.rb
 - test/redis_weighted_group_test.rb
+- test/ruby_content_set_test.rb
+- test/ruby_tag_set_test.rb
+- test/ruby_weighted_group_test.rb
 - test/tests_for_content_sets.rb
 - test/tests_for_tag_sets.rb
 - test/tests_for_weighted_groups.rb
@@ -229,6 +237,9 @@ test_files:
 - test/redis_content_set_test.rb
 - test/redis_tag_set_test.rb
 - test/redis_weighted_group_test.rb
+- test/ruby_content_set_test.rb
+- test/ruby_tag_set_test.rb
+- test/ruby_weighted_group_test.rb
 - test/tests_for_content_sets.rb
 - test/tests_for_tag_sets.rb
 - test/tests_for_weighted_groups.rb