commendo 1.2.4 → 2.0.0

data/lib/commendo/mysql-backed/content_set.rb

@@ -0,0 +1,152 @@
+module Commendo
+  module MySqlBacked
+
+    class ContentSet
+
+      attr_accessor :mysql, :key_base, :tag_set
+
+      DEFAULT_LIMIT = 1000
+
+      def initialize(key_base, tag_set = nil)
+        config_hash = Commendo.config.to_hash
+        config_hash[:flags] = Mysql2::Client::MULTI_STATEMENTS
+        @mysql = Mysql2::Client.new(config_hash)
+        @key_base = key_base
+        @tag_set = tag_set
+      end
+
+      def add_by_group(group, *resources)
+        resources.map! { |r| r.is_a?(Array) ? r : [r, 1] } #sets default score of 1
+        resources.each { |r| add_single(r[0], group, r[1]) }
+      end
+
+      def add(resource, *groups)
+        groups.map! { |g| g.is_a?(Array) ? g : [g, 1] } #sets default score of 1
+        query = add_single_prepared_query
+        groups.each { |(g, s)| query.execute(@key_base, resource, g, s, s) }
+      end
+
+      def add_single(resource, group, score)
+        add(resource, [group, score])
+      end
+
+      def add_and_calculate(resource, *groups)
+        add(resource, *groups)
+      end
+
+      def groups(resource)
+        groups_prepared_query.execute(@key_base, resource).map { |r| r['groupname'] }
+      end
+
+      def delete(resource)
+        delete_prepared_query.execute(@key_base, resource)
+      end
+
+      def calculate_similarity(threshold = nil)
+        threshold = nil if threshold == 0
+        @threshold = threshold
+      end
+
+      def calculate_similarity_for_resource(resource, threshold = 0)
+      end
+
+      def similar_to(resource, limit = DEFAULT_LIMIT)
+        resource = [resource] unless resource.is_a? Array
+        results = @mysql.query(similar_to_query(@key_base, resource, limit)) if @threshold.nil?
+        results = @mysql.query(similar_to_with_threshold_query(@key_base, resource, @threshold, limit)) unless @threshold.nil?
+        similar = results.map { |r| {resource: r['similar'], similarity: r['similarity'].round(3)} }
+        return similar if resource.length == 1
+        grouped = similar.group_by { |r| r[:resource] }
+        grouped.map { |resource, similar| {resource: resource, similarity: similar.inject(0.0) { |sum, s| sum += s[:similarity] }} }.sort_by { |h| [h[:similarity], h[:resource]] }.reverse
+      end
+
+      def filtered_similar_to(resource, options = {})
+        if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?) || @tag_set.empty?
+          return similar_to(resource, options[:limit] || DEFAULT_LIMIT)
+        else
+          similar = similar_to(resource)
+          limit = options[:limit] || similar.length
+          filtered = []
+          similar.each do |s|
+            return filtered if filtered.length >= limit
+            filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
+          end
+          return filtered
+        end
+      end
+
+      def remove_from_groups(resource, *groups)
+        @mysql.query(remove_from_groups_prepared_query(@key_base, resource, groups))
+      end
+
+      def remove_from_groups_and_calculate(resource, *groups)
+        remove_from_groups(resource, *groups)
+        calculate_similarity_for_resource(resource)
+      end
+
+      private
+
+      def add_single_prepared_query
+        @add_single_prepared_query ||= @mysql.prepare('INSERT INTO Resources (keybase, name, groupname, score) VALUES (?,?,?,?) ON DUPLICATE KEY UPDATE score = score + ?')
+      end
+
+      def groups_prepared_query
+        @groups_prepared_query ||= @mysql.prepare('SELECT DISTINCT groupname FROM Resources WHERE keybase=? AND name=?')
+      end
+
+      def delete_prepared_query
+        @delete_prepared_query ||= @mysql.prepare('DELETE FROM Resources WHERE keybase = ? AND name = ?')
+      end
+
+      def remove_from_groups_prepared_query(keybase, name, groups)
+        "
+DELETE FROM Resources WHERE keybase = '#{keybase}' AND name = '#{name}' AND groupname IN (#{groups.map { |r| "'#{r}'" }.join(',')})"
+      end
+
+      def similar_to_query(keybase, resources, limit)
+        "
+SELECT similar, intersect_score, l_union, r_union, intersect_score / (l_union + r_union) AS similarity
+FROM (
+  SELECT r.name AS similar,
+  SUM(l.score + r.score) AS intersect_score,
+  l_us.union_score AS l_union,
+  r_us.union_score AS r_union
+  FROM Resources AS l
+  JOIN Resources AS r ON l.keybase = r.keybase AND l.groupname = r.groupname
+  JOIN UnionScores as l_us ON l_us.keybase = l.keybase AND l_us.name = l.name
+  JOIN UnionScores as r_us ON r_us.keybase = r.keybase AND r_us.name = r.name
+  WHERE l.keybase = '#{keybase}'
+  AND l.name IN (#{resources.map { |r| "'#{r}'" }.join(',')})
+  AND l.name <> r.name
+  GROUP BY l.name, r.name
+) AS similar_resources
+ORDER BY similarity DESC, similar DESC
+LIMIT #{limit}"
+      end
+
+      def similar_to_with_threshold_query(keybase, resources, threshold, limit)
+        "
+SELECT similar, intersect_score, l_union, r_union, similarity FROM (
+  SELECT similar, intersect_score, l_union, r_union, intersect_score / (l_union + r_union) AS similarity FROM (
+    SELECT r.name AS similar,
+    SUM(l.score + r.score) AS intersect_score,
+    (SELECT SUM(score) FROM Resources WHERE keybase = l.keybase AND name = l.name) AS l_union,
+    (SELECT SUM(score) FROM Resources WHERE keybase = r.keybase AND name = r.name) AS r_union
+    FROM Resources AS l
+    JOIN Resources AS r ON l.keybase = r.keybase AND l.groupname = r.groupname
+    WHERE l.keybase = '#{keybase}'
+    AND l.name IN (#{resources.map { |r| "'#{r}'" }.join(',')})
+    AND l.name <> r.name
+    GROUP BY l.name, r.name
+  ) AS similar_resources
+) AS similar
+WHERE similarity > #{threshold}
+ORDER BY similarity DESC, similar DESC
+LIMIT #{limit}"
+      end
+
+
+    end
+
+  end
+end

data/lib/commendo/mysql-backed/tag_set.rb

@@ -0,0 +1,81 @@
+module Commendo
+  module MySqlBacked
+    class TagSet
+
+      attr_accessor :mysql, :key_base
+
+      def initialize(key_base)
+        config_hash = Commendo.config.to_hash
+        @mysql = Mysql2::Client.new(config_hash)
+        @key_base = key_base
+      end
+
+      def empty?
+        result = empty_prepared_query.execute(@key_base)
+        result.count.zero?
+      end
+
+      def get(resource)
+        result = get_tags_prepared_query.execute(@key_base, resource)
+        result.map { |r| r['tag'] }
+      end
+
+      def add(resource, *tags)
+        return if tags.empty?
+        @mysql.transaction do |client|
+          insert_tags(resource, tags)
+        end
+      end
+
+      def set(resource, *tags)
+        @mysql.transaction do |client|
+          delete(resource)
+          insert_tags(resource, tags) unless tags.empty?
+        end
+      end
+
+      def matches(resource, include, exclude = [])
+        resource_tags = get(resource)
+        can_include = include.nil? || include.empty? || (resource_tags & include).length > 0
+        should_exclude = !exclude.nil? && !exclude.empty? && (resource_tags & exclude).length > 0
+        return can_include && !should_exclude
+      end
+
+      def delete(resource, *tags)
+        if tags.empty?
+          delete_all_tags_prepared_query.execute(@key_base, resource)
+        else
+          tags.each { |t| delete_tags_prepared_query.execute(@key_base, resource, t) }
+        end
+      end
+
+      private
+
+      def insert_tags(resource, tags)
+        tags.each { |t| insert_prepared_query.execute(@key_base, resource, t) }
+      end
+
+      def get_tags_prepared_query
+        @get_tags_prepared_query ||= @mysql.prepare('SELECT tag FROM Tags t WHERE keybase = ? AND name = ?')
+      end
+
+      def delete_all_tags_prepared_query
+        @delete_all_tags_prepared_query ||= @mysql.prepare('DELETE FROM Tags WHERE keybase = ? AND name = ?')
+      end
+
+      def delete_tags_prepared_query
+        @delete_tags_prepared_query ||= @mysql.prepare('DELETE FROM Tags WHERE keybase = ? AND name = ? AND tag = ?')
+      end
+
+      def insert_prepared_query
+        @insert_prepared_query ||= @mysql.prepare('INSERT IGNORE INTO Tags (keybase, name, tag) VALUES (?,?,?)')
+      end
+
+      def empty_prepared_query
+        @empty_prepared_query ||= @mysql.prepare('SELECT tag FROM Tags WHERE keybase = ? LIMIT 1')
+      end
+
+    end
+  end
+end
+

data/lib/commendo/mysql-backed/weighted_group.rb

@@ -0,0 +1,40 @@
+module Commendo
+  module MySqlBacked
+
+    class WeightedGroup
+
+      attr_accessor :mysql, :content_sets, :key_base, :tag_set
+
+      def initialize(key_base, *content_sets)
+        config_hash = Commendo.config.to_hash
+        @mysql = Mysql2::Client.new(config_hash)
+        @key_base = key_base
+        @content_sets = content_sets
+      end
+
+      def similar_to(resource, limit = 0)
+        similar = @content_sets.flat_map { |cs| cs[:cs].similar_to(resource).map { |s| {resource: s[:resource], similarity: (s[:similarity] * cs[:weight]).round(3)} } }
+        grouped = similar.group_by { |r| r[:resource] }
+        totaled_similar = grouped.map { |resource, similar| {resource: resource, similarity: similar.inject(0.0) { |sum, s| sum += s[:similarity] }} }.sort_by { |h| [h[:similarity], h[:resource]] }.reverse
+        limit > 0 ? totaled_similar[0..limit-1] : totaled_similar
+      end
+
+      def filtered_similar_to(resource, options = {})
+        if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?)
+          return similar_to(resource, options[:limit] || 0)
+        else
+          similar = similar_to(resource)
+          limit = options[:limit] || similar.length
+          filtered = []
+          similar.each do |s|
+            return filtered if filtered.length >= limit
+            filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
+          end
+          return filtered
+        end
+      end
+
+    end
+
+  end
+end

data/lib/commendo/redis-backed/content_set.rb

@@ -0,0 +1,194 @@
+module Commendo
+  module RedisBacked
+
+    class ContentSet
+
+      attr_accessor :redis, :key_base, :tag_set
+
+      def initialize(key_base, tag_set = nil)
+        @redis = Redis.new(host: Commendo.config.host, port: Commendo.config.port, db: Commendo.config.database, timeout: 120)
+        @key_base = key_base
+        @tag_set = tag_set
+      end
+
+      def add_by_group(group, *resources)
+        resources.each do |resource|
+          if resource.kind_of?(Array)
+            add_single(resource[0], group, resource[1])
+          else
+            add_single(resource, group, 1)
+          end
+        end
+      end
+
+      def add(resource, *groups)
+        groups.each do |group|
+          if group.kind_of?(Array)
+            add_single(resource, group[0], group[1])
+          else
+            add_single(resource, group, 1)
+          end
+        end
+      end
+
+      def add_single(resource, group, score)
+        redis.zincrby(group_key(group), score, resource)
+        redis.zincrby(resource_key(resource), score, group)
+      end
+
+      def add_and_calculate(resource, *groups)
+        add(resource, *groups)
+        calculate_similarity_for_resource(resource, 0)
+      end
+
+      def groups(resource)
+        redis.zrange(resource_key(resource), 0, -1)
+      end
+
+      def delete(resource)
+        similar = similar_to(resource)
+        similar.each do |other_resource|
+          redis.zrem(similarity_key(other_resource[:resource]), "#{resource}")
+        end
+        #TODO delete from groups?
+        redis.del(similarity_key(resource))
+        redis.del(resource_key(resource))
+      end
+
+      SET_TOO_LARGE_FOR_LUA = 999
+
+      def calculate_similarity(threshold = 0)
+        #TODO make this use scan for scaling
+        keys = redis.keys("#{resource_key_base}:*")
+        keys.each_with_index do |key, i|
+          resource = key.gsub(/^#{resource_key_base}:/, '')
+          similarity_key = similarity_key(resource)
+          redis.del(similarity_key)
+          yield(key, i, keys.length) if block_given?
+          completed = redis.eval(similarity_lua, keys: [key], argv: [tmp_key_base, resource_key_base, similar_key_base, group_key_base, threshold])
+          if completed == SET_TOO_LARGE_FOR_LUA
+            calculate_similarity_for_key_resource(key, resource, threshold)
+          end
+        end
+      end
+
+      def calculate_similarity_for_resource(resource, threshold)
+        key = resource_key(resource)
+        calculate_similarity_for_key_resource(key, resource, threshold)
+      end
+
+      def similar_to(resource, limit = 0)
+        finish = limit -1
+        if resource.kind_of? Array
+          keys = resource.map do |res|
+            similarity_key(res)
+          end
+          tmp_key = "#{key_base}:tmp:#{SecureRandom.uuid}"
+          redis.zunionstore(tmp_key, keys)
+          similar_resources = redis.zrevrange(tmp_key, 0, finish, with_scores: true)
+          redis.del(tmp_key)
+        else
+          similar_resources = redis.zrevrange(similarity_key(resource), 0, finish, with_scores: true)
+        end
+        similar_resources.map do |resource|
+          {resource: resource[0], similarity: resource[1].to_f}
+        end
+      end
+
+      def filtered_similar_to(resource, options = {})
+        if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?) || @tag_set.empty?
+          return similar_to(resource, options[:limit] || 0)
+        else
+          similar = similar_to(resource)
+          limit = options[:limit] || similar.length
+          filtered = []
+          similar.each do |s|
+            return filtered if filtered.length >= limit
+            filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
+          end
+          return filtered
+        end
+      end
+
+      def similarity_key(resource)
+        "#{similar_key_base}:#{resource}"
+      end
+
+      def remove_from_groups(resource, *groups)
+        resource_key = resource_key(resource)
+        redis.zrem(resource_key, groups)
+        groups.each do |group|
+          group_key = group_key(group)
+          redis.zrem(group_key, resource)
+        end
+      end
+
+      def remove_from_groups_and_calculate(resource, *groups)
+        remove_from_groups(resource, *groups)
+        calculate_similarity_for_resource(resource, 0)
+      end
+
+      private
+
+      def calculate_similarity_for_key_resource(key, resource, threshold)
+        groups = groups(resource)
+        return if groups.empty?
+        group_keys = groups.map { |group| group_key(group) }
+        tmp_key = "#{tmp_key_base}:#{SecureRandom.uuid}"
+        redis.zunionstore(tmp_key, group_keys)
+        resources = redis.zrange(tmp_key, 0, -1)
+        redis.del(tmp_key)
+        similarity_key = similarity_key(resource)
+        redis.del(similarity_key)
+        resources.each do |to_compare|
+          next if resource == to_compare
+          redis.eval(pair_comparison_lua, keys: [key, resource_key(to_compare), similarity_key(resource), similarity_key(to_compare)], argv: [tmp_key_base, resource, to_compare, threshold])
+        end
+      end
+
+      def similarity_lua
+        @similarity_lua ||= load_similarity_lua
+      end
+
+      def load_similarity_lua
+        file = File.open(File.expand_path('../similarity.lua', __FILE__), "r")
+        file.read
+      end
+
+      def pair_comparison_lua
+        @pair_comparison_lua ||= load_pair_comparison_lua
+      end
+
+      def load_pair_comparison_lua
+        file = File.open(File.expand_path('../pair_comparison.lua', __FILE__), "r")
+        file.read
+      end
+
+      def tmp_key_base
+        "#{key_base}:tmp"
+      end
+
+      def similar_key_base
+        "#{key_base}:similar"
+      end
+
+      def resource_key_base
+        "#{key_base}:resources"
+      end
+
+      def resource_key(resource)
+        "#{resource_key_base}:#{resource}"
+      end
+
+      def group_key_base
+        "#{key_base}:groups"
+      end
+
+      def group_key(group)
+        "#{group_key_base}:#{group}"
+      end
+
+    end
+
+  end
+end