commendo 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ccb7bd9fd0ad412db66aeda26a0048db1d3dc040
+  data.tar.gz: 0afcaf6c33e8eed6b27401e3bce8bf67dc59440a
+SHA512:
+  metadata.gz: 71d5f32a19e6d7de531dc3e99727caafc400eb0d7f0a3aa326f7c03a4b11ed5962654aea49499a75fc1d49344e156987bf20424d778562b901924fa8af939fde
+  data.tar.gz: 12f3145df67a43dd59eb56be2884118becb03c93535e7bb25dffaa4fc0d2a9ddeda6a563ea100c658d752f20567c48099ae8cbd2a8af2cb0dd1e6818780a4e2f

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+.idea
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+# 0.0.1 / 2014-03-28
+* [FEATURE] Initial Release

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in commendo.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Rob Styles
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Commendo
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'commendo'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install commendo
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/commendo/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,54 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+require 'redis'
+require 'commendo'
+
+task :test do
+  $LOAD_PATH.unshift('lib', 'test')
+  Dir.glob('./test/**/*_test.rb') { |f| require f }
+end
+
+task default: :test
+
+def tick i, total = nil
+  if i % 100 == 0
+    print " #{(i / total.to_f).round(2)} " unless total.nil?
+    print '.' if total.nil?
+    $stdout.flush
+  end
+end
+
+task :load_traffic_from_tsv, :filename do |task, args|
+
+  puts "Loading item views from #{args[:filename]}"
+  redis = Redis.new(db: 10, timeout: 30)
+  cs = Commendo::ContentSet.new(redis, 'CommendoScale')
+  redis.flushdb
+  start = Time.now
+  views = []
+  current_resource = nil
+  File.open(args[:filename]) do |f|
+    f.each_line.with_index do |tsv, i|
+      next if i.zero?
+      tick i
+      tsv.chomp!
+      item_type, item_id, user_id, ip_address = tsv.split(/\t/)
+      user_id = user_id != 'NULL' ? user_id : ip_address
+      next if user_id == 'NULL'
+      resource = "#{item_type.gsub(/:+/, '_')}-#{item_id}"
+      if not resource == current_resource
+        cs.add(current_resource, *views) unless (views.empty? || views.length > 100)
+        current_resource = resource
+        views = []
+      end
+      views << user_id unless views.include? user_id
+    end
+  end
+
+  puts 'Processing...'
+  cs.calculate_similarity(0.1999999999) { |key|
+    puts key
+  }
+end
+
+

data/commendo.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'commendo/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'commendo'
+  spec.version       = Commendo::VERSION
+  spec.authors       = ['Rob Styles']
+  spec.email         = ['rob.styles@dynamicorange.com']
+  spec.summary       = 'A Jaccard-similarity recommender using Redis sets'
+  spec.description   = 'A Jaccard-similarity recommender using Redis sets'
+  spec.homepage      = ''
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'redis'
+
+  spec.add_development_dependency 'bundler', '~> 1.5'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'mocha'
+  spec.add_development_dependency 'yard'
+  spec.add_development_dependency 'minitest', '~> 5.0.8'
+end

data/lib/commendo.rb

@@ -0,0 +1,7 @@
+require 'redis'
+require 'commendo/version'
+require 'commendo/content_set'
+
+module Commendo
+  # Your code goes here...
+end

data/lib/commendo/content_set.rb

@@ -0,0 +1,72 @@
+module Commendo
+
+  class ContentSet
+
+    attr_accessor :redis, :key_base
+
+    def initialize(redis, key_base)
+      @redis, @key_base = redis, key_base
+    end
+
+    def add_by_group(group, *resources)
+      resources.each do |res|
+        redis.sadd("#{set_key_base}:#{res}", group)
+      end
+    end
+
+    def add(resource, *groups)
+      redis.sadd("#{set_key_base}:#{resource}", groups)
+    end
+
+    def calculate_similarity(threshold = 0)
+      keys = redis.keys("#{set_key_base}:*")
+      keys.each do |outer_key|
+        outer_res = outer_key.gsub(/^#{set_key_base}:/, '')
+        outer_similarity_key = "#{similar_key_base}:#{outer_res}"
+        calculate_similarity_in_redis(outer_key, outer_similarity_key, threshold)
+        yield(outer_key) if block_given?
+      end
+
+    end
+
+    def calculate_similarity_in_redis(set_key, similiarity_key, threshold)
+      redis.eval(similarity_lua, [set_key, similiarity_key], [set_key_base, threshold])
+    end
+
+    def similar_to(resource)
+      similar = []
+      similar_resources = redis.hgetall("#{similar_key_base}:#{resource}")
+      similar_resources.each do |resource, similarity|
+        similar << {resource: resource, similarity: similarity.to_f}
+      end
+      similar.sort! do |x, y|
+        if y[:similarity] != x[:similarity]
+          y[:similarity] <=> x[:similarity]
+        else
+          y[:resource] <=> x[:resource]
+        end
+      end
+    end
+
+    private
+
+    def similarity_lua
+      @similarity_lua ||= load_similarity_lua
+    end
+
+    def load_similarity_lua
+      file = File.open(File.expand_path('../similarity.lua', __FILE__), "r")
+      file.read
+    end
+
+    def set_key_base
+      "#{key_base}:sets"
+    end
+
+    def similar_key_base
+      "#{key_base}:similar"
+    end
+
+  end
+
+end

data/lib/commendo/similarity.lua

@@ -0,0 +1,31 @@
+local root_key = KEYS[1]
+local sim_key = KEYS[2]
+local set_key_base = ARGV[1]
+local threshold = tonumber(ARGV[2])
+
+redis.log(redis.LOG_NOTICE, 'Running similarity for ' .. root_key)
+local key_matches = redis.call('KEYS', set_key_base .. ':*')
+
+redis.call('DEL', sim_key)
+local count = 0
+-- local similar = {}
+for _,key in ipairs(key_matches) do
+    if key ~= root_key then
+        count = count + 1
+        local intersect = table.getn(redis.call('SINTER', root_key, key))
+        if intersect > 0 then
+            local union = table.getn(redis.call('SUNION', root_key, key))
+            local similarity = intersect / union
+            if similarity > threshold then
+                -- table.insert(similar, key)
+                -- table.insert(similar, similarity)
+                local resource = key:gsub('%' .. set_key_base .. ':', '')
+                redis.call('HSET', sim_key, resource, similarity)
+            end
+        end
+    end
+end
+
+-- redis.call('HMSET', sim_key, unpack(similar))
+redis.log(redis.LOG_NOTICE, 'Finished running similarity for ' .. root_key)
+return true

data/lib/commendo/version.rb

@@ -0,0 +1,3 @@
+module Commendo
+  VERSION = '0.0.1'
+end

data/test/content_set_test.rb

@@ -0,0 +1,100 @@
+gem 'minitest'
+require 'minitest/autorun'
+require 'minitest/pride'
+require 'minitest/mock'
+require 'mocha/setup'
+require 'commendo'
+
+module Commendo
+
+  class ContentSetTest < Minitest::Test
+
+    def test_stores_sets_by_resource
+      redis = Redis.new(db: 15)
+      redis.flushdb
+      key_base = 'CommendoTests'
+      cs = ContentSet.new(redis, key_base)
+      cs.add('resource-1', 'group-1', 'group-2')
+      cs.add('resource-2', 'group-1')
+      cs.add('resource-3', 'group-1', 'group-2')
+      cs.add('resource-4', 'group-2')
+      assert redis.sismember("#{key_base}:sets:resource-1", 'group-1')
+      assert redis.sismember("#{key_base}:sets:resource-2", 'group-1')
+      assert redis.sismember("#{key_base}:sets:resource-3", 'group-1')
+      refute redis.sismember("#{key_base}:sets:resource-4", 'group-1')
+
+      assert redis.sismember("#{key_base}:sets:resource-1", 'group-2')
+      refute redis.sismember("#{key_base}:sets:resource-2", 'group-2')
+      assert redis.sismember("#{key_base}:sets:resource-3", 'group-2')
+      assert redis.sismember("#{key_base}:sets:resource-4", 'group-2')
+    end
+
+    def test_stores_sets_by_group
+      redis = Redis.new(db: 15)
+      redis.flushdb
+      key_base = 'CommendoTests'
+      cs = ContentSet.new(redis, key_base)
+      cs.add_by_group('group-1', 'resource-1', 'resource-2', 'resource-3')
+      cs.add_by_group('group-2', 'resource-1', 'resource-3', 'resource-4')
+      assert redis.sismember("#{key_base}:sets:resource-1", 'group-1')
+      assert redis.sismember("#{key_base}:sets:resource-2", 'group-1')
+      assert redis.sismember("#{key_base}:sets:resource-3", 'group-1')
+      refute redis.sismember("#{key_base}:sets:resource-4", 'group-1')
+
+      assert redis.sismember("#{key_base}:sets:resource-1", 'group-2')
+      refute redis.sismember("#{key_base}:sets:resource-2", 'group-2')
+      assert redis.sismember("#{key_base}:sets:resource-3", 'group-2')
+      assert redis.sismember("#{key_base}:sets:resource-4", 'group-2')
+    end
+
+    def test_calculates_similarity_scores
+      redis = Redis.new(db: 15)
+      redis.flushdb
+      key_base = 'CommendoTests'
+      cs = ContentSet.new(redis, key_base)
+      (3..23).each do |group|
+        (3..23).each do |res|
+          cs.add_by_group(group, res) if res % group == 0
+        end
+      end
+      cs.calculate_similarity
+      expected = [
+        { resource:  '9', similarity: 0.5 },
+        { resource:  '6', similarity: 0.5 },
+        { resource: '12', similarity: 0.33333333333333 },
+        { resource:  '3', similarity: 0.25 },
+        { resource: '21', similarity: 0.16666666666667 },
+        { resource: '15', similarity: 0.16666666666667 }
+      ]
+      assert_equal expected, cs.similar_to(18)
+    end
+
+    def test_calculates_with_threshold
+      redis = Redis.new(db: 15)
+      redis.flushdb
+      key_base = 'CommendoTests'
+      cs = ContentSet.new(redis, key_base)
+      (3..23).each do |group|
+        (3..23).each do |res|
+          cs.add_by_group(group, res) if res % group == 0
+        end
+      end
+      cs.calculate_similarity(0.4)
+      expected = [
+        { resource:  '9', similarity: 0.5 },
+        { resource:  '6', similarity: 0.5 },
+      ]
+      assert_equal expected, cs.similar_to(18)
+    end
+
+    def test_calculate_yields_after_each
+      skip
+    end
+
+    def test_calculate_deletes_old_values_first
+      skip
+    end
+
+  end
+
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: commendo
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Rob Styles
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 5.0.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 5.0.8
+description: A Jaccard-similarity recommender using Redis sets
+email:
+- rob.styles@dynamicorange.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- commendo.gemspec
+- lib/commendo.rb
+- lib/commendo/content_set.rb
+- lib/commendo/similarity.lua
+- lib/commendo/version.rb
+- test/content_set_test.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: A Jaccard-similarity recommender using Redis sets
+test_files:
+- test/content_set_test.rb
+has_rdoc: