disco 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e16ea9f41bc910a0966c4f16f0e48df98f40abb70d2d3b3bd2e8ba2080e57599
-  data.tar.gz: '05490bda394fa0edf02ab33cb16cc65093d5b66b6317671cbdbfc429c3bd196c'
+  metadata.gz: 04d278a7daf8187ac8a5eadaa279c98a0a51a8cf0ad596e793198dcc9141233a
+  data.tar.gz: '0916f7cfb91d5bf48ce1186502f15647c102eba54e07bdc33eb042b75e1fb0c6'
 SHA512:
-  metadata.gz: b12681372323e4bc323915923f91b5883ea97161e2f2a2846548657b25932ddc7ce364e09a6b932506e15e3cb45ff03c8c2cc22022f3a2cfa62d64bbe77e988d
-  data.tar.gz: ea2d4200bfb4d4aed3481ebda8de4133c7af5bb4c4bb7c65685a726bbf661b2285579a507ea464ed49b3609d5350b45404dd6cf99401664e013870a6dee54b7f
+  metadata.gz: a8e977bcf2988e8e4cb85b13959446d068e3a41feeca26f3789ff7aa0a454258340bc81fb3adb470e0143cc6027cd803ef034900cc29db4648b01f855f6cb011
+  data.tar.gz: defc71dd93461a114338f0737cfa3eccae47605e2922aaf12d960a0cb6309131dbba497f7c7d125e962edd055ff7df898cd406544971ed75906cb8c1db6004cf

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.2 (2020-03-26)
+
+- Added experimental `optimize_item_recs` and `optimize_similar_users` methods
+
 ## 0.1.1 (2019-11-14)
 
 - Fixed Rails integration

data/README.md

@@ -4,7 +4,7 @@
 
 - Supports user-based and item-based recommendations
 - Works with explicit and implicit feedback
-- Uses matrix factorization
+- Uses high-performance matrix factorization
 
 [![Build Status](https://travis-ci.org/ankane/disco.svg?branch=master)](https://travis-ci.org/ankane/disco)
 
@@ -202,7 +202,7 @@ recommender = Marshal.load(bin)
 
 ## Algorithms
 
-Disco uses matrix factorization.
+Disco uses high-performance matrix factorization.
 
 - For explicit feedback, it uses [stochastic gradient descent](https://www.csie.ntu.edu.tw/~cjlin/papers/libmf/libmf_journal.pdf)
 - For implicit feedback, it uses [coordinate descent](https://www.csie.ntu.edu.tw/~cjlin/papers/one-class-mf/biased-mf-sdm-with-supp.pdf)
@@ -236,15 +236,44 @@ There are a number of ways to deal with this, but here are some common ones:
 - For user-based recommendations, show new users the most popular items.
 - For item-based recommendations, make content-based recommendations with a gem like [tf-idf-similarity](https://github.com/jpmckinney/tf-idf-similarity).
 
-## Daru
+## Data
 
-Disco works with Daru data frames
+Data can be an array of hashes
 
 ```ruby
-data = Daru::DataFrame.from_csv("ratings.csv")
-recommender.fit(data)
+[{user_id: 1, item_id: 1, rating: 5}, {user_id: 2, item_id: 1, rating: 3}]
+```
+
+Or a Daru data frame
+
+```ruby
+Daru::DataFrame.from_csv("ratings.csv")
 ```
 
+## Faster Similarity [experimental]
+
+If you have a large number of users/items, you can use an approximate nearest neighbors library like [NGT](https://github.com/ankane/ngt) to speed up item-based recommendations and similar users.
+
+Add this line to your application’s Gemfile:
+
+```ruby
+gem 'ngt', '>= 0.2.3'
+```
+
+Speed up item-based recommendations with:
+
+```ruby
+model.optimize_item_recs
+```
+
+Speed up similar users with:
+
+```ruby
+model.optimize_similar_users
+```
+
+This should be called after fitting or loading the model.
+
 ## Reference
 
 Get the global mean

data/lib/disco/recommender.rb

@@ -70,9 +70,11 @@ module Disco
 
       @global_mean = model.bias
 
-      # TODO read from LIBMF directly to Numo for performance
-      @user_factors = Numo::DFloat.cast(model.p_factors)
-      @item_factors = Numo::DFloat.cast(model.q_factors)
+      @user_factors = model.p_factors(format: :numo)
+      @item_factors = model.q_factors(format: :numo)
+
+      @user_index = nil
+      @item_index = nil
     end
 
     def user_recs(user_id, count: 5, item_ids: nil)
@@ -106,17 +108,34 @@ module Disco
       end
     end
 
+    def optimize_similar_items
+      @item_index = create_index(@item_factors)
+    end
+    alias_method :optimize_item_recs, :optimize_similar_items
+
+    def optimize_similar_users
+      @user_index = create_index(@user_factors)
+    end
+
     def similar_items(item_id, count: 5)
-      similar(item_id, @item_map, @item_factors, item_norms, count)
+      similar(item_id, @item_map, @item_factors, item_norms, count, @item_index)
     end
     alias_method :item_recs, :similar_items
 
     def similar_users(user_id, count: 5)
-      similar(user_id, @user_map, @user_factors, user_norms, count)
+      similar(user_id, @user_map, @user_factors, user_norms, count, @user_index)
     end
 
     private
 
+    def create_index(factors)
+      require "ngt"
+
+      index = Ngt::Index.new(factors.shape[1], distance_type: "Cosine")
+      index.batch_insert(factors)
+      index
+    end
+
     def user_norms
       @user_norms ||= norms(@user_factors)
     end
@@ -131,20 +150,37 @@ module Disco
       norms
     end
 
-    def similar(id, map, factors, norms, count)
+    def similar(id, map, factors, norms, count, index)
       i = map[id]
       if i
-        predictions = factors.dot(factors[i, true]) / norms
-
-        predictions =
-          map.keys.zip(predictions).map do |item_id, pred|
-            {item_id: item_id, score: pred}
+        if index && count
+          keys = map.keys
+          result = index.search(factors[i, true], size: count + 1)[1..-1]
+          result.map do |v|
+            {
+              # ids from batch_insert start at 1 instead of 0
+              item_id: keys[v[:id] - 1],
+              # convert cosine distance to cosine similarity
+              score: 1 - v[:distance]
+            }
           end
-
-        predictions.delete_at(i)
-        predictions.sort_by! { |pred| -pred[:score] } # already sorted by id
-        predictions = predictions.first(count) if count
-        predictions
+        else
+          predictions = factors.dot(factors[i, true]) / norms
+
+          predictions =
+            map.keys.zip(predictions).map do |item_id, pred|
+              {item_id: item_id, score: pred}
+            end
+
+          max_score = predictions.delete_at(i)[:score]
+          predictions.sort_by! { |pred| -pred[:score] } # already sorted by id
+          predictions = predictions.first(count) if count
+          # divide by max score to get cosine similarity
+          # only need to do for returned records
+          # could alternatively do cosine distance = 1 - cosine similarity
+          # predictions.each { |pred| pred[:score] /= max_score }
+          predictions
+        end
       else
         []
       end

data/lib/disco/version.rb

@@ -1,3 +1,3 @@
 module Disco
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: disco
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-14 00:00:00.000000000 Z
+date: 2020-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: libmf
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.3
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.3
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: numo-narray
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ngt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.3
 description: 
 email: andrew@chartkick.com
 executables: []
@@ -159,7 +173,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Collaborative filtering for Ruby