disco 0.1.2 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 04d278a7daf8187ac8a5eadaa279c98a0a51a8cf0ad596e793198dcc9141233a
-  data.tar.gz: '0916f7cfb91d5bf48ce1186502f15647c102eba54e07bdc33eb042b75e1fb0c6'
+  metadata.gz: e4a978d2eec39ca280142c49fb4ef4be2e1ad4f35dfa4d977941f46d5d34b466
+  data.tar.gz: 8a29a54bba5ac8b715294e2fce4e34fa1b11442b1800c388807c60b9520ced23
 SHA512:
-  metadata.gz: a8e977bcf2988e8e4cb85b13959446d068e3a41feeca26f3789ff7aa0a454258340bc81fb3adb470e0143cc6027cd803ef034900cc29db4648b01f855f6cb011
-  data.tar.gz: defc71dd93461a114338f0737cfa3eccae47605e2922aaf12d960a0cb6309131dbba497f7c7d125e962edd055ff7df898cd406544971ed75906cb8c1db6004cf
+  metadata.gz: 99376dd48cce340a4fdcb0d76c93b03af494d88167e2caaca0d186fcf5d2303f2524884e0c712c2f8e3d7be79a92b029a8d5fa726bb94826315f283afea0f74b
+  data.tar.gz: eeb8c480098616f93d6c7e39a1bb57e2feefa6af3696c407791ff6f052450eb035f1d1659ded70d7b5fbbbe8cff9f7309118828a454b1d4f9d459321b90035cf

data/CHANGELOG.md

@@ -1,3 +1,34 @@
+## 0.2.4 (2021-02-15)
+
+- Added `user_ids` and `item_ids` methods
+- Added `user_id` argument to `user_factors`
+- Added `item_id` argument to `item_factors`
+
+## 0.2.3 (2020-11-28)
+
+- Added `predict` method
+- Fixed bad recommendations and scores with `user_recs` and explicit feedback
+- Fixed `item_ids` option for `user_recs`
+
+## 0.2.2 (n/a)
+
+- Not available (released by previous gem owner)
+
+## 0.2.1 (2020-10-28)
+
+- Fixed issue with `user_recs` returning rated items
+
+## 0.2.0 (2020-07-31)
+
+- Changed score to always be between -1 and 1 for `item_recs` and `similar_users` (cosine similarity - this makes it easier to understand and consistent with `optimize_item_recs` and `optimize_similar_users`)
+
+## 0.1.3 (2020-06-28)
+
+- Added support for Rover
+- Raise error when missing user or item ids
+- Fixed string keys for Daru data frames
+- `optimize_item_recs` and `optimize_similar_users` methods are no longer experimental
+
 ## 0.1.2 (2020-03-26)
 
 - Added experimental `optimize_item_recs` and `optimize_similar_users` methods

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2019 Andrew Kane
+Copyright (c) 2019-2021 Andrew Kane
 
 MIT License
 

data/README.md

@@ -1,12 +1,12 @@
 # Disco
 
-:fire: Collaborative filtering for Ruby
+:fire: Recommendations for Ruby and Rails using collaborative filtering
 
 - Supports user-based and item-based recommendations
 - Works with explicit and implicit feedback
 - Uses high-performance matrix factorization
 
-[![Build Status](https://travis-ci.org/ankane/disco.svg?branch=master)](https://travis-ci.org/ankane/disco)
+[![Build Status](https://github.com/ankane/disco/workflows/build/badge.svg?branch=master)](https://github.com/ankane/disco/actions)
 
 ## Installation
 
@@ -46,13 +46,13 @@ recommender.fit([
 
 > Use `value` instead of rating for implicit feedback
 
-Get user-based (user-item) recommendations - “users like you also liked”
+Get user-based recommendations - “users like you also liked”
 
 ```ruby
 recommender.user_recs(user_id)
 ```
 
-Get item-based (item-item) recommendations - “users who liked this item also liked”
+Get item-based recommendations - “users who liked this item also liked”
 
 ```ruby
 recommender.item_recs(item_id)
@@ -64,10 +64,10 @@ Use the `count` option to specify the number of recommendations (default is 5)
 recommender.user_recs(user_id, count: 3)
 ```
 
-Get predicted ratings for specific items
+Get predicted ratings for specific users and items
 
 ```ruby
-recommender.user_recs(user_id, item_ids: [1, 2, 3])
+recommender.predict([{user_id: 1, item_id: 2}, {user_id: 2, item_id: 4}])
 ```
 
 Get similar users
@@ -101,14 +101,15 @@ recommender.item_recs("Star Wars (1977)")
 ```ruby
 views = Ahoy::Event.
   where(name: "Viewed post").
-  group(:user_id, "properties->>'post_id'") # postgres syntax
+  group(:user_id).
+  group("properties->>'post_id'"). # postgres syntax
   count
 
 data =
   views.map do |(user_id, post_id), count|
     {
       user_id: user_id,
-      post_id: post_id,
+      item_id: post_id,
       value: count
     }
   end
@@ -244,20 +245,26 @@ Data can be an array of hashes
 [{user_id: 1, item_id: 1, rating: 5}, {user_id: 2, item_id: 1, rating: 3}]
 ```
 
+Or a Rover data frame
+
+```ruby
+Rover.read_csv("ratings.csv")
+```
+
 Or a Daru data frame
 
 ```ruby
 Daru::DataFrame.from_csv("ratings.csv")
 ```
 
-## Faster Similarity [experimental]
+## Faster Similarity
 
 If you have a large number of users/items, you can use an approximate nearest neighbors library like [NGT](https://github.com/ankane/ngt) to speed up item-based recommendations and similar users.
 
 Add this line to your application’s Gemfile:
 
 ```ruby
-gem 'ngt', '>= 0.2.3'
+gem 'ngt', '>= 0.3.0'
 ```
 
 Speed up item-based recommendations with:
@@ -276,19 +283,33 @@ This should be called after fitting or loading the model.
 
 ## Reference
 
+Get ids
+
+```ruby
+recommender.user_ids
+recommender.item_ids
+```
+
 Get the global mean
 
 ```ruby
 recommender.global_mean
 ```
 
-Get the factors
+Get factors
 
 ```ruby
 recommender.user_factors
 recommender.item_factors
 ```
 
+Get factors for specific users and items
+
+```ruby
+recommender.user_factors(user_id)
+recommender.item_factors(item_id)
+```
+
 ## Credits
 
 Thanks to:
@@ -309,3 +330,12 @@ Everyone is encouraged to help improve this project. Here are a few ways you can
 - Fix bugs and [submit pull requests](https://github.com/ankane/disco/pulls)
 - Write, clarify, or fix documentation
 - Suggest or add new features
+
+To get started with development:
+
+```sh
+git clone https://github.com/ankane/disco.git
+cd disco
+bundle install
+bundle exec rake test
+```

data/lib/disco/data.rb

@@ -36,8 +36,7 @@ module Disco
 
       return dest if File.exist?(dest)
 
-      temp_dir ||= File.dirname(Tempfile.new("disco"))
-      temp_path = "#{temp_dir}/#{Time.now.to_f}" # TODO better name
+      temp_path = "#{Dir.tmpdir}/disco-#{Time.now.to_f}" # TODO better name
 
       digest = Digest::SHA2.new
 

data/lib/disco/recommender.rb

@@ -1,38 +1,32 @@
 module Disco
   class Recommender
-    attr_reader :global_mean, :item_factors, :user_factors
+    attr_reader :global_mean
 
     def initialize(factors: 8, epochs: 20, verbose: nil)
       @factors = factors
       @epochs = epochs
       @verbose = verbose
+      @user_map = {}
+      @item_map = {}
     end
 
     def fit(train_set, validation_set: nil)
-      if defined?(Daru)
-        if train_set.is_a?(Daru::DataFrame)
-          train_set = train_set.to_a[0]
-        end
-        if validation_set.is_a?(Daru::DataFrame)
-          validation_set = validation_set.to_a[0]
-        end
-      end
+      train_set = to_dataset(train_set)
+      validation_set = to_dataset(validation_set) if validation_set
 
-      @implicit = !train_set.any? { |v| v[:rating] }
+      check_training_set(train_set)
 
+      @implicit = !train_set.any? { |v| v[:rating] }
       unless @implicit
-        ratings = train_set.map { |o| o[:rating] }
-        check_ratings(ratings)
-        @min_rating = ratings.min
-        @max_rating = ratings.max
+        check_ratings(train_set)
+        @min_rating, @max_rating = train_set.minmax_by { |o| o[:rating] }.map { |o| o[:rating] }
 
         if validation_set
-          check_ratings(validation_set.map { |o| o[:rating] })
+          check_ratings(validation_set)
         end
       end
 
-      check_training_set(train_set)
-      create_maps(train_set)
+      update_maps(train_set)
 
       @rated = Hash.new { |hash, key| hash[key] = {} }
       input = []
@@ -77,12 +71,31 @@ module Disco
       @item_index = nil
     end
 
+    # generates a prediction even if a user has already rated the item
+    def predict(data)
+      data = to_dataset(data)
+
+      u = data.map { |v| @user_map[v[:user_id]] }
+      i = data.map { |v| @item_map[v[:item_id]] }
+
+      new_index = data.each_index.select { |index| u[index].nil? || i[index].nil? }
+      new_index.each do |j|
+        u[j] = 0
+        i[j] = 0
+      end
+
+      predictions = @user_factors[u, true].inner(@item_factors[i, true])
+      predictions.inplace.clip(@min_rating, @max_rating) if @min_rating
+      predictions[new_index] = @global_mean
+      predictions.to_a
+    end
+
     def user_recs(user_id, count: 5, item_ids: nil)
+      check_fit
       u = @user_map[user_id]
 
       if u
-        predictions = @global_mean + @item_factors.dot(@user_factors[u, true])
-        predictions.inplace.clip(@min_rating, @max_rating) if @min_rating
+        predictions = @item_factors.inner(@user_factors[u, true])
 
         predictions =
           @item_map.keys.zip(predictions).map do |item_id, pred|
@@ -91,15 +104,24 @@ module Disco
 
         if item_ids
           idx = item_ids.map { |i| @item_map[i] }.compact
-          predictions.values_at(*idx)
+          predictions = predictions.values_at(*idx)
         else
-          @rated[u].keys.each do |i|
+          @rated[u].keys.sort_by { |v| -v }.each do |i|
             predictions.delete_at(i)
           end
         end
 
         predictions.sort_by! { |pred| -pred[:score] } # already sorted by id
         predictions = predictions.first(count) if count && !item_ids
+
+        # clamp *after* sorting
+        # also, only needed for returned predictions
+        if @min_rating
+          predictions.each do |pred|
+            pred[:score] = pred[:score].clamp(@min_rating, @max_rating)
+          end
+        end
+
         predictions
       else
         # no items if user is unknown
@@ -109,21 +131,51 @@ module Disco
     end
 
     def optimize_similar_items
+      check_fit
       @item_index = create_index(@item_factors)
     end
     alias_method :optimize_item_recs, :optimize_similar_items
 
     def optimize_similar_users
+      check_fit
       @user_index = create_index(@user_factors)
     end
 
     def similar_items(item_id, count: 5)
-      similar(item_id, @item_map, @item_factors, item_norms, count, @item_index)
+      check_fit
+      similar(item_id, @item_map, @item_factors, @item_index ? nil : item_norms, count, @item_index)
     end
     alias_method :item_recs, :similar_items
 
     def similar_users(user_id, count: 5)
-      similar(user_id, @user_map, @user_factors, user_norms, count, @user_index)
+      check_fit
+      similar(user_id, @user_map, @user_factors, @user_index ? nil : user_norms, count, @user_index)
+    end
+
+    def user_ids
+      @user_map.keys
+    end
+
+    def item_ids
+      @item_map.keys
+    end
+
+    def user_factors(user_id = nil)
+      if user_id
+        u = @user_map[user_id]
+        @user_factors[u, true] if u
+      else
+        @user_factors
+      end
+    end
+
+    def item_factors(item_id = nil)
+      if item_id
+        i = @item_map[item_id]
+        @item_factors[i, true] if i
+      else
+        @item_factors
+      end
     end
 
     private
@@ -131,8 +183,11 @@ module Disco
     def create_index(factors)
       require "ngt"
 
+      # could speed up search with normalized cosine
+      # https://github.com/yahoojapan/NGT/issues/36
       index = Ngt::Index.new(factors.shape[1], distance_type: "Cosine")
-      index.batch_insert(factors)
+      ids = index.batch_insert(factors)
+      raise "Unexpected ids. Please report a bug." if ids.first != 1 || ids.last != factors.shape[0]
       index
     end
 
@@ -145,7 +200,7 @@ module Disco
     end
 
     def norms(factors)
-      norms = Numo::DFloat::Math.sqrt((factors * factors).sum(axis: 1))
+      norms = Numo::SFloat::Math.sqrt((factors * factors).sum(axis: 1))
       norms[norms.eq(0)] = 1e-10 # no zeros
       norms
     end
@@ -165,20 +220,21 @@ module Disco
             }
           end
         else
-          predictions = factors.dot(factors[i, true]) / norms
+          # cosine similarity without norms[i]
+          # otherwise, denominator would be (norms[i] * norms)
+          predictions = factors.inner(factors[i, true]) / norms
 
           predictions =
             map.keys.zip(predictions).map do |item_id, pred|
               {item_id: item_id, score: pred}
             end
 
-          max_score = predictions.delete_at(i)[:score]
+          predictions.delete_at(i)
           predictions.sort_by! { |pred| -pred[:score] } # already sorted by id
           predictions = predictions.first(count) if count
-          # divide by max score to get cosine similarity
+          # divide by norms[i] to get cosine similarity
           # only need to do for returned records
-          # could alternatively do cosine distance = 1 - cosine similarity
-          # predictions.each { |pred| pred[:score] /= max_score }
+          predictions.each { |pred| pred[:score] /= norms[i] }
           predictions
         end
       else
@@ -186,19 +242,21 @@ module Disco
       end
     end
 
-    def create_maps(train_set)
-      user_ids = train_set.map { |v| v[:user_id] }.uniq.sort
-      item_ids = train_set.map { |v| v[:item_id] }.uniq.sort
+    def update_maps(train_set)
+      raise ArgumentError, "Missing user_id" if train_set.any? { |v| v[:user_id].nil? }
+      raise ArgumentError, "Missing item_id" if train_set.any? { |v| v[:item_id].nil? }
 
-      @user_map = user_ids.zip(user_ids.size.times).to_h
-      @item_map = item_ids.zip(item_ids.size.times).to_h
+      train_set.each do |v|
+        @user_map[v[:user_id]] ||= @user_map.size
+        @item_map[v[:item_id]] ||= @item_map.size
+      end
     end
 
     def check_ratings(ratings)
-      unless ratings.all? { |r| !r.nil? }
+      unless ratings.all? { |r| !r[:rating].nil? }
         raise ArgumentError, "Missing ratings"
       end
-      unless ratings.all? { |r| r.is_a?(Numeric) }
+      unless ratings.all? { |r| r[:rating].is_a?(Numeric) }
         raise ArgumentError, "Ratings must be numeric"
       end
     end
@@ -207,6 +265,29 @@ module Disco
       raise ArgumentError, "No training data" if train_set.empty?
     end
 
+    def check_fit
+      raise "Not fit" unless defined?(@implicit)
+    end
+
+    def to_dataset(dataset)
+      if defined?(Rover::DataFrame) && dataset.is_a?(Rover::DataFrame)
+        # convert keys to symbols
+        dataset = dataset.dup
+        dataset.keys.each do |k, v|
+          dataset[k.to_sym] ||= dataset.delete(k)
+        end
+        dataset.to_a
+      elsif defined?(Daru::DataFrame) && dataset.is_a?(Daru::DataFrame)
+        # convert keys to symbols
+        dataset = dataset.dup
+        new_names = dataset.vectors.to_a.map { |k| [k, k.to_sym] }.to_h
+        dataset.rename_vectors!(new_names)
+        dataset.to_a[0]
+      else
+        dataset
+      end
+    end
+
     def marshal_dump
       obj = {
         implicit: @implicit,

data/lib/disco/version.rb

@@ -1,3 +1,3 @@
 module Disco
-  VERSION = "0.1.2"
+  VERSION = "0.2.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: disco
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.4
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-26 00:00:00.000000000 Z
+date: 2021-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: libmf
@@ -38,106 +38,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
-- !ruby/object:Gem::Dependency
-  name: activerecord
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: sqlite3
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: daru
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: ngt
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.2.3
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.2.3
-description: 
-email: andrew@chartkick.com
+description:
+email: andrew@ankane.org
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -158,7 +60,7 @@ homepage: https://github.com/ankane/disco
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -173,8 +75,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
-summary: Collaborative filtering for Ruby
+summary: Recommendations for Ruby and Rails using collaborative filtering
 test_files: []