commendo 1.2.4 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f4346b4b053e4b910efdd819476fae6e18db6b45
-  data.tar.gz: faeee86f961b6e576888accd047b6adfe4dcd54e
+  metadata.gz: 04c833f06518b70ff1370c1685df37294c8fd2b9
+  data.tar.gz: dd99d2a4bcca958d2c216d3084b88298c59f462f
 SHA512:
-  metadata.gz: ad4356a6a14f35713ab2eee837b69e78e6ed7995c4131968ce8356df45630fc290817a670a2aa29135f02ff9a9a18928437b3a5263c551065912c5689ebb280d
-  data.tar.gz: b4db90f586213d1174810081d1e18335572f97c51ca351ffd11f3ea395562b9674781375e2c62d8cc16c0926932dc8479a273e556f38b669acf4f2866250cf9a
+  metadata.gz: 9936b22c4e2ffc54e63b36cc13c8dce2d1559115956092c3061ca65e3f5db76302128d5fe46aefb30685394bc1f508a6ca05455bc3e0f2e1ae2ddd6fe9121e3b
+  data.tar.gz: 952208db4416f43325d581f36363a124b0614d656c3b90d38d8b6576350cb8f561aa14022c6084b50b47cc51825a405350682bb6cbb80974590a3110412e0c11

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# 2.0.0 / 2015-12-11
+* [FEATURE] Commendo can now use MySQL as a backend if you prefer it to Redis
+
 # 1.2.4 / 2015-12-09
 * [BUGFIX] Fixed bug with TagSet#empty? reporting empty when only one key present
 

data/bin/commendo-create-mysql-db

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+mysql -h $1 -u $2 -p$3 -e "CREATE SCHEMA IF NOT EXISTS $4 DEFAULT CHARACTER SET utf8;"
+mysql -h $1 -u $2 -p$3 $4 < $(dirname $0)/commendo-create.sql

data/bin/commendo-create.sql

@@ -0,0 +1,99 @@
+SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;
+SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;
+SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='TRADITIONAL,ALLOW_INVALID_DATES';
+
+-- -----------------------------------------------------
+-- Schema commendo
+-- -----------------------------------------------------
+# CREATE SCHEMA IF NOT EXISTS commendo_created DEFAULT CHARACTER SET utf8 ;
+# USE commendo_created ;
+
+-- -----------------------------------------------------
+-- Table `Resources`
+-- -----------------------------------------------------
+CREATE TABLE IF NOT EXISTS `Resources` (
+  `id` INT NOT NULL AUTO_INCREMENT,
+  `keybase` VARCHAR(64) NOT NULL,
+  `name` VARCHAR(128) NOT NULL,
+  `groupname` VARCHAR(128) NOT NULL,
+  `score` FLOAT NOT NULL,
+  `union_score` FLOAT NULL,
+  PRIMARY KEY (`id`),
+  INDEX `name` (`name` ASC),
+  INDEX `groupname` (`groupname` ASC),
+  UNIQUE INDEX `keybase-name-groupname` (`keybase` ASC, `name` ASC, `groupname` ASC),
+  INDEX `keybase` (`keybase` ASC),
+  INDEX `keybase-name-score` (`keybase` ASC, `name` ASC, `score` ASC),
+  INDEX `keybase-groupname` (`keybase` ASC, `groupname` ASC))
+ENGINE = InnoDB;
+
+
+-- -----------------------------------------------------
+-- Table `Tags`
+-- -----------------------------------------------------
+CREATE TABLE IF NOT EXISTS `Tags` (
+  `id` INT NOT NULL AUTO_INCREMENT,
+  `keybase` VARCHAR(64) NOT NULL,
+  `name` VARCHAR(128) NOT NULL,
+  `tag` VARCHAR(64) NOT NULL,
+  PRIMARY KEY (`id`),
+  INDEX `tag` (`tag` ASC),
+  UNIQUE INDEX `keybase-name-tag` (`keybase` ASC, `name` ASC, `tag` ASC),
+  INDEX `keybase` (`keybase` ASC),
+  INDEX `name` (`name` ASC))
+ENGINE = InnoDB;
+
+
+-- -----------------------------------------------------
+-- Table `UnionScores`
+-- -----------------------------------------------------
+CREATE TABLE IF NOT EXISTS `UnionScores` (
+  `id` INT NOT NULL AUTO_INCREMENT,
+  `keybase` VARCHAR(64) NOT NULL,
+  `name` VARCHAR(128) NOT NULL,
+  `union_score` FLOAT NOT NULL,
+  PRIMARY KEY (`id`),
+  INDEX `keybase` (`keybase` ASC),
+  INDEX `name` (`name` ASC),
+  UNIQUE INDEX `keybase-name` (`keybase` ASC, `name` ASC))
+ENGINE = InnoDB;
+
+# USE @schema_name;
+
+DELIMITER $$
+# USE @schema_name$$
+CREATE DEFINER = CURRENT_USER
+TRIGGER `Resources_AFTER_INSERT`
+AFTER INSERT ON `Resources` FOR EACH ROW
+BEGIN
+  SET @union_score = (
+    SELECT SUM(score)
+    FROM Resources
+    WHERE keybase = new.keybase
+    AND name = new.name
+  );
+  INSERT INTO UnionScores (keybase, name, union_score)
+  VALUES (new.keybase, new.name, @union_score)
+  ON DUPLICATE KEY UPDATE union_score = @union_score;
+END$$
+
+# USE @schema_name$$
+CREATE DEFINER = CURRENT_USER TRIGGER `Resources_AFTER_UPDATE` AFTER UPDATE ON `Resources` FOR EACH ROW
+BEGIN
+  SET @union_score = (
+    SELECT SUM(score)
+    FROM Resources
+    WHERE keybase = new.keybase
+    AND name = new.name
+  );
+  UPDATE UnionScores SET union_score = @union_score
+  WHERE keybase = new.keybase
+  AND name = new.name;
+END$$
+
+
+DELIMITER ;
+
+SET SQL_MODE=@OLD_SQL_MODE;
+SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS;
+SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS;

data/bin/commendo-load-tsv

@@ -5,13 +5,17 @@ require 'commendo'
 require 'progressbar'
 
 filename = ARGV[0]
-redis_db = ARGV[1].to_i
-base_key = ARGV[2]
 
-redis = Redis.new(db: redis_db, timeout: 60)
-cs = Commendo::ContentSet.new(redis, base_key)
+Commendo.config do |config|
+  config.backend = :redis
+  config.host = 'localhost'
+  config.port = 6379
+  config.database = 15
+end
+Redis.new(host: Commendo.config.host, port: Commendo.config.port, db: Commendo.config.database).flushdb
+cs = Commendo::ContentSet.new(key_base: 'MeducationViews')
 
-puts "Loading."
+puts 'Loading.'
 file_length = `wc -l #{filename}`.to_i
 pbar = ProgressBar.new('Loading TSV file', file_length)
 File.open(filename) do |f|
@@ -22,6 +26,7 @@ File.open(filename) do |f|
     cs.add(resource, *ids)
   end
 end
+pbar.finish
 puts "\nFinished loading"
 
 puts 'Calculating similarities'
@@ -30,3 +35,4 @@ cs.calculate_similarity do |key, i, total|
   pbar ||= ProgressBar.new('Calculating similarity', total)
   pbar.inc
 end
+pbar.finish

data/bin/commendo-load-tsv-mysql.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+require 'redis'
+require 'commendo'
+require 'progressbar'
+
+filename = ARGV[0]
+key_base = ARGV[1]
+
+Commendo.config do |config|
+  config.backend = :mysql
+  config.host = 'localhost'
+  config.port = 3306
+  config.database = 'commendo_test'
+  config.username = 'commendo'
+  config.password = 'commendo123'
+end
+client = Mysql2::Client.new(Commendo.config.to_hash)
+client.query("DELETE FROM Resources WHERE keybase='#{key_base}';")
+cs = Commendo::ContentSet.new(key_base: key_base)
+
+puts 'Loading.'
+file_length = `wc -l #{filename}`.to_i
+pbar = ProgressBar.new('Loading TSV file', file_length)
+File.open(filename) do |f|
+  f.each_line do |line|
+    pbar.inc
+    ids = line.strip.split("\t")
+    resource = ids.shift
+    cs.add(resource, *ids)
+  end
+end
+pbar.finish
+puts "\nFinished loading"
+
+# puts 'Calculating similarities'
+# # pbar = nil
+# cs.calculate_similarity do |key, i, total|
+#   pbar ||= ProgressBar.new('Calculating similarity', total)
+#   # pbar.inc
+#   $stderr.puts key
+# end
+# pbar.finish

data/bin/commendo-time-mysql.rb

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+require 'redis'
+require 'commendo'
+require 'progressbar'
+
+key_base = ARGV[0]
+limit = ARGV[1].to_i
+
+Commendo.config do |config|
+  config.backend = :mysql
+  config.host = 'localhost'
+  config.port = 3306
+  config.database = 'commendo_test'
+  config.username = 'commendo'
+  config.password = 'commendo123'
+end
+cs = Commendo::ContentSet.new(key_base: key_base)
+
+$stderr.puts "Selecting #{limit} random names to use"
+client = Mysql2::Client.new(Commendo.config.to_hash)
+names_to_query = client.query("SELECT DISTINCT name FROM Resources WHERE keybase = '#{key_base}' ORDER BY RAND() LIMIT #{limit}")
+names_to_query = names_to_query.map { |r| r['name'] }
+
+pbar = ProgressBar.new('Querying similar_to', names_to_query.length)
+names_to_query.each do |name|
+  cs.similar_to(name)
+  pbar.inc
+end
+pbar.finish
+

data/commendo.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |spec|
   spec.version       = Commendo::VERSION
   spec.authors       = ['Rob Styles']
   spec.email         = ['rob.styles@dynamicorange.com']
-  spec.summary       = 'A Jaccard-similarity recommender using Redis sets'
-  spec.description   = 'A Jaccard-similarity recommender using Redis sets'
+  spec.summary       = 'A Jaccard-similarity recommender using Redis sets or MySQL'
+  spec.description   = 'A Jaccard-similarity recommender using Redis sets or MySQL'
   spec.homepage      = ''
   spec.license       = 'MIT'
 
@@ -19,7 +19,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'redis'
+  spec.add_dependency 'mysql2'
   spec.add_dependency 'progressbar'
+  spec.add_dependency 'slop'
 
   spec.add_development_dependency 'bundler', '~> 1.5'
   spec.add_development_dependency 'rake'

data/lib/commendo.rb

@@ -1,9 +1,33 @@
+require 'forwardable'
+require 'mysql2'
+require_relative 'mysql2/client'
 require 'redis'
+
+require 'commendo/configuration'
 require 'commendo/version'
 require 'commendo/content_set'
 require 'commendo/tag_set'
 require 'commendo/weighted_group'
 
+require 'commendo/redis-backed/content_set'
+require 'commendo/redis-backed/tag_set'
+require 'commendo/redis-backed/weighted_group'
+
+require 'commendo/mysql-backed/content_set'
+require 'commendo/mysql-backed/tag_set'
+require 'commendo/mysql-backed/weighted_group'
+
 module Commendo
 
+  def self.config
+    config = @@config ||= Configuration.new
+    yield(config) if block_given?
+    config
+  end
+
+  def self.config=(config)
+    raise 'Configuration must be either a Commendo::Configuration object or nil to reset' unless config.nil? || config.is_a?(Configuration)
+    @@config = config
+  end
+
 end

data/lib/commendo/configuration.rb

@@ -0,0 +1,25 @@
+module Commendo
+  class Configuration
+
+    attr_accessor :backend, :host, :port, :database, :username, :password
+
+    def initialize
+      @backend = :redis
+      @host = 'localhost'
+      @port = 6379
+      @database = 15
+    end
+
+    def to_hash
+      {
+          backend: backend,
+          host: host,
+          port: port,
+          database: database,
+          username: username,
+          password: password
+      }
+    end
+
+  end
+end

data/lib/commendo/content_set.rb

@@ -1,191 +1,22 @@
 module Commendo
 
   class ContentSet
+    extend Forwardable
 
-    attr_accessor :redis, :key_base, :tag_set
+    def_delegators :@backend,
+                   :add_by_group, :add, :add_single, :add_and_calculate,
+                   :groups, :delete,
+                   :calculate_similarity, :calculate_similarity_for_resource, :calculate_similarity_for_key_resource,
+                   :similar_to, :filtered_similar_to,
+                   :similarity_key,
+                   :remove_from_groups, :remove_from_groups_and_calculate
 
-    def initialize(redis, key_base, tag_set = nil)
-      @redis, @key_base, @tag_set = redis, key_base, tag_set
-    end
-
-    def add_by_group(group, *resources)
-      resources.each do |resource|
-        if resource.kind_of?(Array)
-          add_single(resource[0], group, resource[1])
-        else
-          add_single(resource, group, 1)
-        end
-      end
-    end
-
-    def add(resource, *groups)
-      groups.each do |group|
-        if group.kind_of?(Array)
-          add_single(resource, group[0], group[1])
-        else
-          add_single(resource, group, 1)
-        end
-      end
-    end
-
-    def add_single(resource, group, score)
-      redis.zincrby(group_key(group), score, resource)
-      redis.zincrby(resource_key(resource), score, group)
-    end
-
-    def add_and_calculate(resource, *groups)
-      add(resource, *groups)
-      calculate_similarity_for_resource(resource, 0)
-    end
-
-    def groups(resource)
-      redis.zrange(resource_key(resource), 0, -1)
-    end
-
-    def delete(resource)
-      similar = similar_to(resource)
-      similar.each do |other_resource|
-        redis.zrem(similarity_key(other_resource[:resource]), "#{resource}")
-      end
-      #TODO delete from groups?
-      redis.del(similarity_key(resource))
-      redis.del(resource_key(resource))
-    end
-
-    SET_TOO_LARGE_FOR_LUA = 999
-
-    def calculate_similarity(threshold = 0)
-      #TODO make this use scan for scaling
-      keys = redis.keys("#{resource_key_base}:*")
-      keys.each_with_index do |key, i|
-        resource = key.gsub(/^#{resource_key_base}:/, '')
-        similarity_key = similarity_key(resource)
-        redis.del(similarity_key)
-        yield(key, i, keys.length) if block_given?
-        completed = redis.eval(similarity_lua, keys: [key], argv: [tmp_key_base, resource_key_base, similar_key_base, group_key_base, threshold])
-        if completed == SET_TOO_LARGE_FOR_LUA
-          calculate_similarity_for_key_resource(key, resource, threshold)
-        end
-      end
-    end
-
-
-    def calculate_similarity_for_resource(resource, threshold)
-      key = resource_key(resource)
-      calculate_similarity_for_key_resource(key, resource, threshold)
-    end
-
-    def calculate_similarity_for_key_resource(key, resource, threshold)
-      groups = groups(resource)
-      return if groups.empty?
-      group_keys = groups.map { |group| group_key(group) }
-      tmp_key = "#{tmp_key_base}:#{SecureRandom.uuid}"
-      redis.zunionstore(tmp_key, group_keys)
-      resources = redis.zrange(tmp_key, 0, -1)
-      redis.del(tmp_key)
-      similarity_key = similarity_key(resource)
-      redis.del(similarity_key)
-      resources.each do |to_compare|
-        next if resource == to_compare
-        redis.eval(pair_comparison_lua, keys: [key, resource_key(to_compare), similarity_key(resource), similarity_key(to_compare)], argv: [tmp_key_base, resource, to_compare, threshold])
-      end
-    end
-
-    def similar_to(resource, limit = 0)
-      finish = limit -1
-      if resource.kind_of? Array
-        keys = resource.map do |res|
-          similarity_key(res)
-        end
-        tmp_key = "#{key_base}:tmp:#{SecureRandom.uuid}"
-        redis.zunionstore(tmp_key, keys)
-        similar_resources = redis.zrevrange(tmp_key, 0, finish, with_scores: true)
-        redis.del(tmp_key)
-      else
-        similar_resources = redis.zrevrange(similarity_key(resource), 0, finish, with_scores: true)
-      end
-      similar_resources.map do |resource|
-        {resource: resource[0], similarity: resource[1].to_f}
-      end
-    end
-
-    def filtered_similar_to(resource, options = {})
-      if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?) || @tag_set.empty?
-        return similar_to(resource, options[:limit] || 0)
-      else
-        similar = similar_to(resource)
-        limit = options[:limit] || similar.length
-        filtered = []
-        similar.each do |s|
-          return filtered if filtered.length >= limit
-          filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
-        end
-        return filtered
-      end
-    end
-
-    def similarity_key(resource)
-      "#{similar_key_base}:#{resource}"
-    end
-
-    def remove_from_groups(resource, *groups)
-      resource_key = resource_key(resource)
-      redis.zrem(resource_key, groups)
-      groups.each do |group|
-        group_key = group_key(group)
-        redis.zrem(group_key, resource)
-      end
-    end
-
-    def remove_from_groups_and_calculate(resource, *groups)
-      remove_from_groups(resource, *groups)
-      calculate_similarity_for_resource(resource, 0)
-    end
-
-    private
-
-    def similarity_lua
-      @similarity_lua ||= load_similarity_lua
-    end
-
-    def load_similarity_lua
-      file = File.open(File.expand_path('../similarity.lua', __FILE__), "r")
-      file.read
-    end
-
-    def pair_comparison_lua
-      @pair_comparison_lua ||= load_pair_comparison_lua
-    end
-
-    def load_pair_comparison_lua
-      file = File.open(File.expand_path('../pair_comparison.lua', __FILE__), "r")
-      file.read
-    end
-
-    def tmp_key_base
-      "#{key_base}:tmp"
-    end
-
-    def similar_key_base
-      "#{key_base}:similar"
-    end
-
-    def resource_key_base
-      "#{key_base}:resources"
-    end
-
-    def resource_key(resource)
-      "#{resource_key_base}:#{resource}"
-    end
-
-    def group_key_base
-      "#{key_base}:groups"
-    end
-
-    def group_key(group)
-      "#{group_key_base}:#{group}"
+    def initialize(args)
+      @backend = RedisBacked::ContentSet.new(args[:key_base], args[:tag_set]) if Commendo.config.backend == :redis
+      @backend = MySqlBacked::ContentSet.new(args[:key_base], args[:tag_set]) if Commendo.config.backend == :mysql
+      raise 'Unrecognised backend type, try :redis or :mysql' if @backend.nil?
     end
 
   end
 
-end
+end