disco 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8f0494a12f2efe7d077d989960b4f40bae39e9d433b8c38d45be367f79ad1a3e
-  data.tar.gz: d5ef0e761f5cc3a409e0b7c3d8cdc14a33d5c03ca4eca7b476159c82954ad163
+  metadata.gz: 1ed4e5ff1d50d49068cfc9dc6b26df6be24aca1827b3f3724c3e351b06d768ac
+  data.tar.gz: 24756addf552956497889a9898eb9f93cdeec996c2a8a3cc1808c61425ea3d39
 SHA512:
-  metadata.gz: 0a4995986ac209da39ff9f7c449225f1fc560364936f6df76647e33500f8f07749dc857f2a334b7bb59dadd941403feeee4f43f51647a7df799051a095f0cb3b
-  data.tar.gz: e1bef5d6f9d3272cbca8a6fc56d7d14bcc2bc2b269ddda7931a256f6a4c019419b2e590e06841e5a09914b9523a5b4c09c9ff8bfd817b3d3bd2e99ecf3368752
+  metadata.gz: 50d5a9ba262c8c751f77fb2037235d9186e7616af55c653518752b2300d0c0b1a70d59dd935f0eaed9e1bd09881e547c66446dd0af8bfe641a679acae76fddd0
+  data.tar.gz: 00b260249dbc2831ad4948d531176c53df965b28f317b0e46987c5be7209da63126033cf75116b01f78ceefd8da977a9fb14bf014e67ae71e18bb634a9852335

data/CHANGELOG.md

@@ -1,3 +1,26 @@
-## 0.1.0
+## 0.2.1 (2020-10-28)
+
+- Fixed issue with `user_recs` returning rated items
+
+## 0.2.0 (2020-07-31)
+
+- Changed score to always be between -1 and 1 for `item_recs` and `similar_users` (cosine similarity - this makes it easier to understand and consistent with `optimize_item_recs` and `optimize_similar_users`)
+
+## 0.1.3 (2020-06-28)
+
+- Added support for Rover
+- Raise error when missing user or item ids
+- Fixed string keys for Daru data frames
+- `optimize_item_recs` and `optimize_similar_users` methods are no longer experimental
+
+## 0.1.2 (2020-03-26)
+
+- Added experimental `optimize_item_recs` and `optimize_similar_users` methods
+
+## 0.1.1 (2019-11-14)
+
+- Fixed Rails integration
+
+## 0.1.0 (2019-11-14)
 
 - First release

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2019 Andrew Kane
+Copyright (c) 2019-2020 Andrew Kane
 
 MIT License
 

data/README.md

@@ -1,10 +1,10 @@
 # Disco
 
-:fire: Collaborative filtering for Ruby
+:fire: Recommendations for Ruby and Rails using collaborative filtering
 
 - Supports user-based and item-based recommendations
 - Works with explicit and implicit feedback
-- Uses matrix factorization
+- Uses high-performance matrix factorization
 
 [![Build Status](https://travis-ci.org/ankane/disco.svg?branch=master)](https://travis-ci.org/ankane/disco)
 
@@ -101,14 +101,15 @@ recommender.item_recs("Star Wars (1977)")
 ```ruby
 views = Ahoy::Event.
   where(name: "Viewed post").
-  group(:user_id, "properties->>'post_id'") # postgres syntax
+  group(:user_id).
+  group("properties->>'post_id'"). # postgres syntax
   count
 
 data =
   views.map do |(user_id, post_id), count|
     {
       user_id: user_id,
-      post_id: post_id,
+      item_id: post_id,
       value: count
     }
   end
@@ -202,7 +203,7 @@ recommender = Marshal.load(bin)
 
 ## Algorithms
 
-Disco uses matrix factorization.
+Disco uses high-performance matrix factorization.
 
 - For explicit feedback, it uses [stochastic gradient descent](https://www.csie.ntu.edu.tw/~cjlin/papers/libmf/libmf_journal.pdf)
 - For implicit feedback, it uses [coordinate descent](https://www.csie.ntu.edu.tw/~cjlin/papers/one-class-mf/biased-mf-sdm-with-supp.pdf)
@@ -236,15 +237,50 @@ There are a number of ways to deal with this, but here are some common ones:
 - For user-based recommendations, show new users the most popular items.
 - For item-based recommendations, make content-based recommendations with a gem like [tf-idf-similarity](https://github.com/jpmckinney/tf-idf-similarity).
 
-## Daru
+## Data
 
-Disco works with Daru data frames
+Data can be an array of hashes
 
 ```ruby
-data = Daru::DataFrame.from_csv("ratings.csv")
-recommender.fit(data)
+[{user_id: 1, item_id: 1, rating: 5}, {user_id: 2, item_id: 1, rating: 3}]
+```
+
+Or a Rover data frame
+
+```ruby
+Rover.read_csv("ratings.csv")
 ```
 
+Or a Daru data frame
+
+```ruby
+Daru::DataFrame.from_csv("ratings.csv")
+```
+
+## Faster Similarity
+
+If you have a large number of users/items, you can use an approximate nearest neighbors library like [NGT](https://github.com/ankane/ngt) to speed up item-based recommendations and similar users.
+
+Add this line to your application’s Gemfile:
+
+```ruby
+gem 'ngt', '>= 0.3.0'
+```
+
+Speed up item-based recommendations with:
+
+```ruby
+model.optimize_item_recs
+```
+
+Speed up similar users with:
+
+```ruby
+model.optimize_similar_users
+```
+
+This should be called after fitting or loading the model.
+
 ## Reference
 
 Get the global mean
@@ -280,3 +316,12 @@ Everyone is encouraged to help improve this project. Here are a few ways you can
 - Fix bugs and [submit pull requests](https://github.com/ankane/disco/pulls)
 - Write, clarify, or fix documentation
 - Suggest or add new features
+
+To get started with development:
+
+```sh
+git clone https://github.com/ankane/disco.git
+cd disco
+bundle install
+bundle exec rake test
+```

data/app/models/disco/recommendation.rb

@@ -0,0 +1,8 @@
+module Disco
+  class Recommendation < ActiveRecord::Base
+    self.table_name = "disco_recommendations"
+
+    belongs_to :subject, polymorphic: true
+    belongs_to :item, polymorphic: true
+  end
+end

data/lib/disco/data.rb

@@ -36,8 +36,7 @@ module Disco
 
       return dest if File.exist?(dest)
 
-      temp_dir ||= File.dirname(Tempfile.new("disco"))
-      temp_path = "#{temp_dir}/#{Time.now.to_f}" # TODO better name
+      temp_path = "#{Dir.tmpdir}/disco-#{Time.now.to_f}" # TODO better name
 
       digest = Digest::SHA2.new
 

data/lib/disco/recommender.rb

@@ -9,14 +9,8 @@ module Disco
     end
 
     def fit(train_set, validation_set: nil)
-      if defined?(Daru)
-        if train_set.is_a?(Daru::DataFrame)
-          train_set = train_set.to_a[0]
-        end
-        if validation_set.is_a?(Daru::DataFrame)
-          validation_set = validation_set.to_a[0]
-        end
-      end
+      train_set = to_dataset(train_set)
+      validation_set = to_dataset(validation_set) if validation_set
 
       @implicit = !train_set.any? { |v| v[:rating] }
 
@@ -70,9 +64,11 @@ module Disco
 
       @global_mean = model.bias
 
-      # TODO read from LIBMF directly to Numo for performance
-      @user_factors = Numo::DFloat.cast(model.p_factors)
-      @item_factors = Numo::DFloat.cast(model.q_factors)
+      @user_factors = model.p_factors(format: :numo)
+      @item_factors = model.q_factors(format: :numo)
+
+      @user_index = nil
+      @item_index = nil
     end
 
     def user_recs(user_id, count: 5, item_ids: nil)
@@ -91,7 +87,7 @@ module Disco
           idx = item_ids.map { |i| @item_map[i] }.compact
           predictions.values_at(*idx)
         else
-          @rated[u].keys.each do |i|
+          @rated[u].keys.sort_by { |v| -v }.each do |i|
             predictions.delete_at(i)
           end
         end
@@ -106,17 +102,34 @@ module Disco
       end
     end
 
+    def optimize_similar_items
+      @item_index = create_index(@item_factors)
+    end
+    alias_method :optimize_item_recs, :optimize_similar_items
+
+    def optimize_similar_users
+      @user_index = create_index(@user_factors)
+    end
+
     def similar_items(item_id, count: 5)
-      similar(item_id, @item_map, @item_factors, item_norms, count)
+      similar(item_id, @item_map, @item_factors, item_norms, count, @item_index)
     end
     alias_method :item_recs, :similar_items
 
     def similar_users(user_id, count: 5)
-      similar(user_id, @user_map, @user_factors, user_norms, count)
+      similar(user_id, @user_map, @user_factors, user_norms, count, @user_index)
     end
 
     private
 
+    def create_index(factors)
+      require "ngt"
+
+      index = Ngt::Index.new(factors.shape[1], distance_type: "Cosine")
+      index.batch_insert(factors)
+      index
+    end
+
     def user_norms
       @user_norms ||= norms(@user_factors)
     end
@@ -126,25 +139,41 @@ module Disco
     end
 
     def norms(factors)
-      norms = Numo::DFloat::Math.sqrt((factors * factors).sum(axis: 1))
+      norms = Numo::SFloat::Math.sqrt((factors * factors).sum(axis: 1))
       norms[norms.eq(0)] = 1e-10 # no zeros
       norms
     end
 
-    def similar(id, map, factors, norms, count)
+    def similar(id, map, factors, norms, count, index)
       i = map[id]
       if i
-        predictions = factors.dot(factors[i, true]) / norms
-
-        predictions =
-          map.keys.zip(predictions).map do |item_id, pred|
-            {item_id: item_id, score: pred}
+        if index && count
+          keys = map.keys
+          result = index.search(factors[i, true], size: count + 1)[1..-1]
+          result.map do |v|
+            {
+              # ids from batch_insert start at 1 instead of 0
+              item_id: keys[v[:id] - 1],
+              # convert cosine distance to cosine similarity
+              score: 1 - v[:distance]
+            }
           end
-
-        predictions.delete_at(i)
-        predictions.sort_by! { |pred| -pred[:score] } # already sorted by id
-        predictions = predictions.first(count) if count
-        predictions
+        else
+          predictions = factors.dot(factors[i, true]) / norms
+
+          predictions =
+            map.keys.zip(predictions).map do |item_id, pred|
+              {item_id: item_id, score: pred}
+            end
+
+          max_score = predictions.delete_at(i)[:score]
+          predictions.sort_by! { |pred| -pred[:score] } # already sorted by id
+          predictions = predictions.first(count) if count
+          # divide by max score to get cosine similarity
+          # only need to do for returned records
+          predictions.each { |pred| pred[:score] /= max_score }
+          predictions
+        end
       else
         []
       end
@@ -154,6 +183,9 @@ module Disco
       user_ids = train_set.map { |v| v[:user_id] }.uniq.sort
       item_ids = train_set.map { |v| v[:item_id] }.uniq.sort
 
+      raise ArgumentError, "Missing user_id" if user_ids.any?(&:nil?)
+      raise ArgumentError, "Missing item_id" if item_ids.any?(&:nil?)
+
       @user_map = user_ids.zip(user_ids.size.times).to_h
       @item_map = item_ids.zip(item_ids.size.times).to_h
     end
@@ -171,6 +203,25 @@ module Disco
       raise ArgumentError, "No training data" if train_set.empty?
     end
 
+    def to_dataset(dataset)
+      if defined?(Rover::DataFrame) && dataset.is_a?(Rover::DataFrame)
+        # convert keys to symbols
+        dataset = dataset.dup
+        dataset.keys.each do |k, v|
+          dataset[k.to_sym] ||= dataset.delete(k)
+        end
+        dataset.to_a
+      elsif defined?(Daru::DataFrame) && dataset.is_a?(Daru::DataFrame)
+        # convert keys to symbols
+        dataset = dataset.dup
+        new_names = dataset.vectors.to_a.map { |k| [k, k.to_sym] }.to_h
+        dataset.rename_vectors!(new_names)
+        dataset.to_a[0]
+      else
+        dataset
+      end
+    end
+
     def marshal_dump
       obj = {
         implicit: @implicit,

data/lib/disco/version.rb

@@ -1,3 +1,3 @@
 module Disco
-  VERSION = "0.1.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: disco
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-14 00:00:00.000000000 Z
+date: 2020-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: libmf
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.3
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.3
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: numo-narray
   requirement: !ruby/object:Gem::Requirement
@@ -122,7 +122,35 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+- !ruby/object:Gem::Dependency
+  name: rover-df
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ngt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+description:
 email: andrew@chartkick.com
 executables: []
 extensions: []
@@ -131,6 +159,7 @@ files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- app/models/disco/recommendation.rb
 - lib/disco.rb
 - lib/disco/data.rb
 - lib/disco/engine.rb
@@ -143,7 +172,7 @@ homepage: https://github.com/ankane/disco
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -158,8 +187,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
-summary: Collaborative filtering for Ruby
+summary: Recommendations for Ruby and Rails using collaborative filtering
 test_files: []