cmfrec 0.1.1 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2091fae505c3d98468c6cfa08e0f80c6e97804fe8558bea97191336fa0179423
-  data.tar.gz: b66cece9f8659a6f79ac6ee302a1d2068d9a8b89b8903b2182f0fa8be6838869
+  metadata.gz: 7a200b99780aeee83c5a0190e593c3806c4140aa4c096b0ef2c112fd21a858b7
+  data.tar.gz: 27b354dd491ca1d7a728d4ef2318c72c56d76c691286bc267c74d3b8dafd7c5b
 SHA512:
-  metadata.gz: 5feed4c89f6249646b61d0713bcbc83725561942437ba3abdf2ca70a82a8f76d4a614b4f47a2c07e158a7155442f31b730b1d759234520bd34e50cae7e723b16
-  data.tar.gz: 649ce14693b2b7cfcc6039d4e39d94c3e3dce695e32b62f6c8c94c2751621622c519e5b536c917446e781b1acbadd3a511789efaae3ffc4aa2062738697f975f
+  metadata.gz: 0c1ceefeac9131a804d94b6da78c7a6614f50c15d144841277829e5279ce2583872a681e972e336e21cb82d49466dee9f5f1de6a482fe8a99ab7aa5176ab0e5c
+  data.tar.gz: 930b20b017b92555071699b38d903230da3ca1fc7b8b91b9e2f96ae3ae4dda9c8c289a47117beaa1e2539e5c9d622fe958d431f5106e8ab7c4809ea17e9fb6e8

data/CHANGELOG.md

@@ -1,3 +1,23 @@
+## 0.1.5 (2021-08-10)
+
+- Fixed issue with `user_recs` and `new_user_recs` returning rated items
+- Fixed error with `new_user_recs`
+
+## 0.1.4 (2021-02-04)
+
+- Added support for saving and loading recommenders
+- Added `similar_users` and `similar_items`
+- Improved ARM detection
+
+## 0.1.3 (2020-12-28)
+
+- Added ARM shared library for Mac
+
+## 0.1.2 (2020-12-09)
+
+- Added `load_movielens` method
+- Updated cmfrec to 2.4.1
+
 ## 0.1.1 (2020-11-28)
 
 - Added `predict` method

data/LICENSE.txt

@@ -1,7 +1,7 @@
 MIT License
 
 Copyright (c) 2020 David Cortes
-Copyright (c) 2020 Andrew Kane
+Copyright (c) 2020-2021 Andrew Kane
 
 All rights reserved.
 

data/README.md

@@ -16,7 +16,7 @@ Add this line to your application’s Gemfile:
 gem 'cmfrec'
 ```
 
-Not available for Windows yet
+For Windows, also follow [these instructions](#windows-installation).
 
 ## Getting Started
 
@@ -107,6 +107,73 @@ Get recommendations with only side information
 recommender.new_user_recs([], user_info: {cats: 0, dogs: 2})
 ```
 
+## Similarity
+
+Add this line to your application’s Gemfile:
+
+```ruby
+gem 'ngt'
+```
+
+Get similar users
+
+```ruby
+recommender.similar_users(user_id)
+```
+
+Get similar items - “users who liked this item also liked”
+
+```ruby
+recommender.similar_items(item_id)
+```
+
+## Examples
+
+### MovieLens
+
+Load the data
+
+```ruby
+ratings, user_info, item_info = Cmfrec.load_movielens
+```
+
+Create a recommender and get predictions
+
+```ruby
+recommender = Cmfrec::Recommender.new(factors: 20)
+recommender.fit(ratings.first(80000), user_info: user_info, item_info: item_info)
+recommender.predict(ratings.last(20000))
+```
+
+### Ahoy
+
+[Ahoy](https://github.com/ankane/ahoy) is a great source for implicit feedback
+
+```ruby
+views = Ahoy::Event.
+  where(name: "Viewed post").
+  group(:user_id).
+  group("properties->>'post_id'"). # postgres syntax
+  count
+
+data =
+  views.map do |(user_id, post_id), count|
+    {
+      user_id: user_id,
+      item_id: post_id,
+      value: count
+    }
+  end
+```
+
+Create a recommender and get recommended posts for a user
+
+```ruby
+recommender = Cmfrec::Recommender.new
+recommender.fit(data)
+recommender.user_recs(current_user.id)
+```
+
 ## Options
 
 Specify the number of factors and epochs
@@ -145,6 +212,24 @@ Or a Rover data frame
 Rover.read_csv("ratings.csv")
 ```
 
+## Storing Recommenders
+
+Store the recommender
+
+```ruby
+bin = Marshal.dump(recommender)
+File.binwrite("recommender.bin", bin)
+```
+
+> You can save it to a file, database, or any other storage system
+
+Load a recommender
+
+```ruby
+bin = File.binread("recommender.bin")
+recommender = Marshal.load(bin)
+```
+
 ## Reference
 
 Get the global mean
@@ -167,6 +252,14 @@ recommender.user_bias
 recommender.item_bias
 ```
 
+## Windows Installation
+
+On Windows, build the [cmfrec C shared library](https://github.com/david-cortes/cmfrec#instalation) and set:
+
+```ruby
+Cmfrec.ffi_lib = "path/to/cmfrec.dll"
+```
+
 ## History
 
 View the [changelog](https://github.com/ankane/cmfrec/blob/master/CHANGELOG.md)

data/lib/cmfrec.rb

@@ -3,12 +3,15 @@ require "etc"
 require "fiddle/import"
 
 # modules
+require "cmfrec/data"
 require "cmfrec/recommender"
 require "cmfrec/version"
 
 module Cmfrec
   class Error < StandardError; end
 
+  extend Data
+
   class << self
     attr_accessor :ffi_lib
   end
@@ -16,7 +19,11 @@ module Cmfrec
     if Gem.win_platform?
       "cmfrec.dll"
     elsif RbConfig::CONFIG["host_os"] =~ /darwin/i
-      "libcmfrec.dylib"
+      if RbConfig::CONFIG["host_cpu"] =~ /arm/i
+        "libcmfrec.arm64.dylib"
+      else
+        "libcmfrec.dylib"
+      end
     else
       "libcmfrec.so"
     end

data/lib/cmfrec/data.rb

@@ -0,0 +1,100 @@
+module Cmfrec
+  module Data
+    def load_movielens
+      require "csv"
+
+      data_path = download_file("ml-100k/u.data", "http://files.grouplens.org/datasets/movielens/ml-100k/u.data",
+        file_hash: "06416e597f82b7342361e41163890c81036900f418ad91315590814211dca490")
+      user_path = download_file("ml-100k/u.user", "http://files.grouplens.org/datasets/movielens/ml-100k/u.user",
+        file_hash: "f120e114da2e8cf314fd28f99417c94ae9ddf1cb6db8ce0e4b5995d40e90e62c")
+      item_path = download_file("ml-100k/u.item", "http://files.grouplens.org/datasets/movielens/ml-100k/u.item",
+        file_hash: "553841ebc7de3a0fd0d6b62a204ea30c1e651aacfb2814c7a6584ac52f2c5701")
+
+      # convert u.item to utf-8
+      movies_str = File.read(item_path).encode("UTF-8", "binary", invalid: :replace, undef: :replace, replace: "")
+
+      user_info = []
+      CSV.foreach(user_path, col_sep: "|") do |row|
+        user = {user_id: row[0].to_i}
+        10.times do |i|
+          user[:"region#{i}"] = row[4][0] == i.to_s ? 1 : 0
+        end
+        user_info << user
+      end
+
+      item_info = []
+      movies = {}
+      genres = %w(unknown action adventure animation childrens comedy crime documentary drama fantasy filmnoir horror musical mystery romance scifi thriller war western)
+      CSV.parse(movies_str, col_sep: "|", converters: [:numeric]) do |row|
+        movies[row[0]] = row[1]
+        item = {item_id: row[1], year: row[2] ? Date.parse(row[2]).year : 1970}
+        genres.each_with_index do |genre, i|
+          item[:"genre_#{genre}"] = row[i + 5]
+        end
+        item_info << item
+      end
+
+      data = []
+      CSV.foreach(data_path, col_sep: "\t", converters: [:numeric]) do |row|
+        data << {
+          user_id: row[0],
+          item_id: movies[row[1]],
+          rating: row[2]
+        }
+      end
+
+      [data, user_info, item_info]
+    end
+
+    private
+
+    def download_file(fname, origin, file_hash:)
+      require "fileutils"
+
+      # TODO handle this better
+      raise "No HOME" unless ENV["HOME"]
+      dest = "#{ENV["HOME"]}/.cmfrec/#{fname}"
+      FileUtils.mkdir_p(File.dirname(dest))
+
+      return dest if File.exist?(dest)
+
+      require "digest"
+      require "net/http"
+      require "tmpdir"
+
+      temp_path = "#{Dir.tmpdir}/cmfrec-#{Time.now.to_f}" # TODO better name
+
+      digest = Digest::SHA2.new
+
+      uri = URI(origin)
+
+      # Net::HTTP automatically adds Accept-Encoding for compression
+      # of response bodies and automatically decompresses gzip
+      # and deflateresponses unless a Range header was sent.
+      # https://ruby-doc.org/stdlib-2.6.4/libdoc/net/http/rdoc/Net/HTTP.html
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
+        request = Net::HTTP::Get.new(uri)
+
+        puts "Downloading data from #{origin}"
+        File.open(temp_path, "wb") do |f|
+          http.request(request) do |response|
+            response.read_body do |chunk|
+              f.write(chunk)
+              digest.update(chunk)
+            end
+          end
+        end
+      end
+
+      if digest.hexdigest != file_hash
+        raise Error, "Bad hash: #{digest.hexdigest}"
+      end
+
+      puts "Hash verified: #{file_hash}"
+
+      FileUtils.mv(temp_path, dest)
+
+      dest
+    end
+  end
+end

data/lib/cmfrec/ffi.rb

@@ -10,17 +10,19 @@ module Cmfrec
       raise e
     end
 
+    # https://github.com/david-cortes/cmfrec/blob/master/src/cmfrec.h
+
     typealias "bool", "char"
     # determined by CMakeLists.txt
     typealias "int_t", "int"
     typealias "real_t", "double"
 
-    extern "int_t fit_collective_explicit_als(real_t *biasA, real_t *biasB, real_t *A, real_t *B, real_t *C, real_t *D, real_t *Ai, real_t *Bi, bool add_implicit_features, bool reset_values, int_t seed, real_t *glob_mean, real_t *U_colmeans, real_t *I_colmeans, int_t m, int_t n, int_t k, int_t X_row[], int_t X_col[], real_t *X, size_t nnz, real_t *Xfull, real_t *weight, bool user_bias, bool item_bias, real_t lam, real_t *lam_unique, real_t *U, int_t m_u, int_t p, real_t *II, int_t n_i, int_t q, int_t U_row[], int_t U_col[], real_t *U_sp, size_t nnz_U, int_t I_row[], int_t I_col[], real_t *I_sp, size_t nnz_I, bool NA_as_zero_X, bool NA_as_zero_U, bool NA_as_zero_I, int_t k_main, int_t k_user, int_t k_item, real_t w_main, real_t w_user, real_t w_item, real_t w_implicit, int_t niter, int_t nthreads, bool verbose, bool handle_interrupt, bool use_cg, int_t max_cg_steps, bool finalize_chol, bool nonneg, int_t max_cd_steps, bool nonneg_C, bool nonneg_D, bool precompute_for_predictions, bool include_all_X, real_t *B_plus_bias, real_t *precomputedBtB, real_t *precomputedTransBtBinvBt, real_t *precomputedBeTBeChol, real_t *precomputedBiTBi, real_t *precomputedTransCtCinvCt, real_t *precomputedCtCw)"
-    extern "int_t fit_collective_implicit_als(real_t *A, real_t *B, real_t *C, real_t *D, bool reset_values, int_t seed, real_t *U_colmeans, real_t *I_colmeans, int_t m, int_t n, int_t k, int_t X_row[], int_t X_col[], real_t *X, size_t nnz, real_t lam, real_t *lam_unique, real_t *U, int_t m_u, int_t p, real_t *II, int_t n_i, int_t q, int_t U_row[], int_t U_col[], real_t *U_sp, size_t nnz_U, int_t I_row[], int_t I_col[], real_t *I_sp, size_t nnz_I, bool NA_as_zero_U, bool NA_as_zero_I, int_t k_main, int_t k_user, int_t k_item, real_t w_main, real_t w_user, real_t w_item, real_t *w_main_multiplier, real_t alpha, bool adjust_weight, bool apply_log_transf, int_t niter, int_t nthreads, bool verbose, bool handle_interrupt, bool use_cg, int_t max_cg_steps, bool finalize_chol, bool nonneg, int_t max_cd_steps, bool nonneg_C, bool nonneg_D, bool precompute_for_predictions, real_t *precomputedBtB, real_t *precomputedBeTBe, real_t *precomputedBeTBeChol)"
-    extern "int_t factors_collective_explicit_single(real_t *a_vec, real_t *a_bias, real_t *u_vec, int_t p, real_t *u_vec_sp, int_t u_vec_X_col[], size_t nnz_u_vec, real_t *u_bin_vec, int_t pbin, bool NA_as_zero_U, bool NA_as_zero_X, bool nonneg, real_t *C, real_t *Cb, real_t glob_mean, real_t *biasB, real_t *U_colmeans, real_t *Xa, int_t X_col[], size_t nnz, real_t *Xa_dense, int_t n, real_t *weight, real_t *B, real_t *Bi, bool add_implicit_features, int_t k, int_t k_user, int_t k_item, int_t k_main, real_t lam, real_t *lam_unique, real_t w_main, real_t w_user, real_t w_implicit, int_t n_max, bool include_all_X, real_t *TransBtBinvBt, real_t *BtB, real_t *BeTBeChol, real_t *BiTBi, real_t *CtCw, real_t *TransCtCinvCt, real_t *B_plus_bias)"
-    extern "int_t factors_collective_implicit_single(real_t *a_vec, real_t *u_vec, int_t p, real_t *u_vec_sp, int_t u_vec_X_col[], size_t nnz_u_vec, bool NA_as_zero_U, bool nonneg, real_t *U_colmeans, real_t *B, int_t n, real_t *C, real_t *Xa, int_t X_col[], size_t nnz, int_t k, int_t k_user, int_t k_item, int_t k_main, real_t lam, real_t alpha, real_t w_main, real_t w_user, real_t w_main_multiplier, bool apply_log_transf, real_t *BeTBe, real_t *BtB, real_t *BeTBeChol)"
-    extern "void predict_multiple(real_t *restrict A, int_t k_user, real_t *restrict B, int_t k_item, real_t *restrict biasA, real_t *restrict biasB, real_t glob_mean, int_t k, int_t k_main, int_t m, int_t n, int_t predA[], int_t predB[], size_t nnz, real_t *restrict outp, int_t nthreads)"
+    extern "int_t fit_collective_explicit_als(real_t *restrict biasA, real_t *restrict biasB, real_t *restrict A, real_t *restrict B, real_t *restrict C, real_t *restrict D, real_t *restrict Ai, real_t *restrict Bi, bool add_implicit_features, bool reset_values, int_t seed, real_t *restrict glob_mean, real_t *restrict U_colmeans, real_t *restrict I_colmeans, int_t m, int_t n, int_t k, int_t ixA[], int_t ixB[], real_t *restrict X, size_t nnz, real_t *restrict Xfull, real_t *restrict weight, bool user_bias, bool item_bias, bool center, real_t lam, real_t *restrict lam_unique, real_t l1_lam, real_t *restrict l1_lam_unique, bool scale_lam, bool scale_lam_sideinfo, real_t *restrict U, int_t m_u, int_t p, real_t *restrict II, int_t n_i, int_t q, int_t U_row[], int_t U_col[], real_t *restrict U_sp, size_t nnz_U, int_t I_row[], int_t I_col[], real_t *restrict I_sp, size_t nnz_I, bool NA_as_zero_X, bool NA_as_zero_U, bool NA_as_zero_I, int_t k_main, int_t k_user, int_t k_item, real_t w_main, real_t w_user, real_t w_item, real_t w_implicit, int_t niter, int_t nthreads, bool verbose, bool handle_interrupt, bool use_cg, int_t max_cg_steps, bool finalize_chol, bool nonneg, int_t max_cd_steps, bool nonneg_C, bool nonneg_D, bool precompute_for_predictions, bool include_all_X, real_t *restrict B_plus_bias, real_t *restrict precomputedBtB, real_t *restrict precomputedTransBtBinvBt, real_t *restrict precomputedBtXbias, real_t *restrict precomputedBeTBeChol, real_t *restrict precomputedBiTBi, real_t *restrict precomputedTransCtCinvCt, real_t *restrict precomputedCtCw)"
+    extern "int_t fit_collective_implicit_als(real_t *restrict A, real_t *restrict B, real_t *restrict C, real_t *restrict D, bool reset_values, int_t seed, real_t *restrict U_colmeans, real_t *restrict I_colmeans, int_t m, int_t n, int_t k, int_t ixA[], int_t ixB[], real_t *restrict X, size_t nnz, real_t lam, real_t *restrict lam_unique, real_t l1_lam, real_t *restrict l1_lam_unique, real_t *restrict U, int_t m_u, int_t p, real_t *restrict II, int_t n_i, int_t q, int_t U_row[], int_t U_col[], real_t *restrict U_sp, size_t nnz_U, int_t I_row[], int_t I_col[], real_t *restrict I_sp, size_t nnz_I, bool NA_as_zero_U, bool NA_as_zero_I, int_t k_main, int_t k_user, int_t k_item, real_t w_main, real_t w_user, real_t w_item, real_t *restrict w_main_multiplier, real_t alpha, bool adjust_weight, bool apply_log_transf, int_t niter, int_t nthreads, bool verbose, bool handle_interrupt, bool use_cg, int_t max_cg_steps, bool finalize_chol, bool nonneg, int_t max_cd_steps, bool nonneg_C, bool nonneg_D, bool precompute_for_predictions, real_t *restrict precomputedBtB, real_t *restrict precomputedBeTBe, real_t *restrict precomputedBeTBeChol)"
+    extern "int_t factors_collective_explicit_single(real_t *restrict a_vec, real_t *restrict a_bias,real_t *restrict u_vec, int_t p,real_t *restrict u_vec_sp, int_t u_vec_ixB[], size_t nnz_u_vec,real_t *restrict u_bin_vec, int_t pbin,bool NA_as_zero_U, bool NA_as_zero_X,bool nonneg,real_t *restrict C, real_t *restrict Cb,real_t glob_mean, real_t *restrict biasB,real_t *restrict U_colmeans,real_t *restrict Xa, int_t ixB[], size_t nnz,real_t *restrict Xa_dense, int_t n,real_t *restrict weight,real_t *restrict B,real_t *restrict Bi, bool add_implicit_features,int_t k, int_t k_user, int_t k_item, int_t k_main,real_t lam, real_t *restrict lam_unique,real_t l1_lam, real_t *restrict l1_lam_unique,bool scale_lam, bool scale_lam_sideinfo,real_t w_main, real_t w_user, real_t w_implicit,int_t n_max, bool include_all_X,real_t *restrict BtB,real_t *restrict TransBtBinvBt,real_t *restrict BtXbias,real_t *restrict BeTBeChol,real_t *restrict BiTBi,real_t *restrict CtCw,real_t *restrict TransCtCinvCt,real_t *restrict B_plus_bias)"
+    extern "int_t factors_collective_implicit_single(real_t *restrict a_vec,real_t *restrict u_vec, int_t p,real_t *restrict u_vec_sp, int_t u_vec_ixB[], size_t nnz_u_vec,bool NA_as_zero_U,bool nonneg,real_t *restrict U_colmeans,real_t *restrict B, int_t n, real_t *restrict C,real_t *restrict Xa, int_t ixB[], size_t nnz,int_t k, int_t k_user, int_t k_item, int_t k_main,real_t lam, real_t l1_lam, real_t alpha, real_t w_main, real_t w_user,real_t w_main_multiplier,bool apply_log_transf,real_t *restrict BeTBe,real_t *restrict BtB,real_t *restrict BeTBeChol)"
     extern "int_t predict_X_old_collective_explicit(int_t row[], int_t col[], real_t *restrict predicted, size_t n_predict, real_t *restrict A, real_t *restrict biasA, real_t *restrict B, real_t *restrict biasB, real_t glob_mean, int_t k, int_t k_user, int_t k_item, int_t k_main, int_t m, int_t n_max, int_t nthreads)"
+    extern "int_t predict_X_old_collective_implicit(int_t row[], int_t col[], real_t *restrict predicted, size_t n_predict, real_t *restrict A, real_t *restrict B, int_t k, int_t k_user, int_t k_item, int_t k_main, int_t m, int_t n, int_t nthreads)"
     extern "int_t topN(real_t *restrict a_vec, int_t k_user, real_t *restrict B, int_t k_item, real_t *restrict biasB, real_t glob_mean, real_t biasA, int_t k, int_t k_main, int_t *restrict include_ix, int_t n_include, int_t *restrict exclude_ix, int_t n_exclude, int_t *restrict outp_ix, real_t *restrict outp_score, int_t n_top, int_t n, int_t nthreads)"
   end
 end

data/lib/cmfrec/recommender.rb

@@ -11,29 +11,199 @@ module Cmfrec
         item_bias: item_bias,
         add_implicit_features: add_implicit_features
       )
+
+      @fit = false
+      @user_map = {}
+      @item_map = {}
+      @user_info_map = {}
+      @item_info_map = {}
     end
 
     def fit(train_set, user_info: nil, item_info: nil)
+      reset
+      partial_fit(train_set, user_info: user_info, item_info: item_info)
+    end
+
+    def predict(data)
+      check_fit
+
+      data = to_dataset(data)
+
+      u = data.map { |v| @user_map[v[:user_id]] || @user_map.size }
+      i = data.map { |v| @item_map[v[:item_id]] || @item_map.size }
+
+      row = int_ptr(u)
+      col = int_ptr(i)
+      n_predict = data.size
+      predicted = Fiddle::Pointer.malloc(n_predict * Fiddle::SIZEOF_DOUBLE)
+
+      if @implicit
+        check_status FFI.predict_X_old_collective_implicit(
+          row, col, predicted, n_predict,
+          @a, @b,
+          @k, @k_user, @k_item, @k_main,
+          @m, @n,
+          @nthreads
+        )
+      else
+        check_status FFI.predict_X_old_collective_explicit(
+          row, col, predicted, n_predict,
+          @a, @bias_a,
+          @b, @bias_b,
+          @global_mean,
+          @k, @k_user, @k_item, @k_main,
+          @m, @n,
+          @nthreads
+        )
+      end
+
+      predictions = real_array(predicted)
+      predictions.map! { |v| v.nan? ? @global_mean : v } if @implicit
+      predictions
+    end
+
+    def user_recs(user_id, count: 5, item_ids: nil)
+      check_fit
+      user = @user_map[user_id]
+
+      if user
+        # TODO use top_n for item_ids as well
+        if item_ids
+          # remove missing ids
+          item_ids = item_ids.select { |v| @item_map[v] }
+
+          data = item_ids.map { |v| {user_id: user_id, item_id: v} }
+          scores = predict(data)
+
+          item_ids.zip(scores).map do |item_id, score|
+            {item_id: item_id, score: score}
+          end
+        else
+          a_vec = @a[user * @k * Fiddle::SIZEOF_DOUBLE, @k * Fiddle::SIZEOF_DOUBLE]
+          a_bias = @bias_a ? @bias_a[user * Fiddle::SIZEOF_DOUBLE, Fiddle::SIZEOF_DOUBLE].unpack1("d") : 0
+          # @rated[user] will be nil for recommenders saved before 0.1.5
+          top_n(a_vec: a_vec, a_bias: a_bias, count: count, rated: (@rated[user] || {}).keys)
+        end
+      else
+        # no items if user is unknown
+        # TODO maybe most popular items
+        []
+      end
+    end
+
+    # TODO add item_ids
+    def new_user_recs(data, count: 5, user_info: nil)
+      check_fit
+
+      a_vec, a_bias, rated = factors_warm(data, user_info: user_info)
+      top_n(a_vec: a_vec, a_bias: a_bias, count: count, rated: rated)
+    end
+
+    def user_factors
+      read_factors(@a, [@m, @m_u].max, @k_user + @k + @k_main)
+    end
+
+    def item_factors
+      read_factors(@b, [@n, @n_i].max, @k_item + @k + @k_main)
+    end
+
+    def user_bias
+      read_bias(@bias_a) if @bias_a
+    end
+
+    def item_bias
+      read_bias(@bias_b) if @bias_b
+    end
+
+    def similar_items(item_id, count: 5)
+      check_fit
+      similar(item_id, @item_map, item_factors, count, item_index)
+    end
+    alias_method :item_recs, :similar_items
+
+    def similar_users(user_id, count: 5)
+      check_fit
+      similar(user_id, @user_map, user_factors, count, user_index)
+    end
+
+    private
+
+    def user_index
+      @user_index ||= create_index(user_factors)
+    end
+
+    def item_index
+      @item_index ||= create_index(item_factors)
+    end
+
+    def create_index(factors)
+      require "ngt"
+
+      index = Ngt::Index.new(@k, distance_type: "Cosine")
+      index.batch_insert(factors)
+      index
+    end
+
+    # TODO include bias
+    def similar(id, map, factors, count, index)
+      i = map[id]
+      if i
+        keys = map.keys
+        result = index.search(factors[i], size: count + 1)[1..-1]
+        result.map do |v|
+          {
+            # ids from batch_insert start at 1 instead of 0
+            item_id: keys[v[:id] - 1],
+            # convert cosine distance to cosine similarity
+            score: 1 - v[:distance]
+          }
+        end
+      else
+        []
+      end
+    end
+
+    def reset
+      @fit = false
+      @user_map.clear
+      @item_map.clear
+      @user_info_map.clear
+      @item_info_map.clear
+      @user_index = nil
+      @item_index = nil
+    end
+
+    # TODO resize pointers as needed and reset values for new memory
+    def partial_fit(train_set, user_info: nil, item_info: nil)
       train_set = to_dataset(train_set)
 
-      @implicit = !train_set.any? { |v| v[:rating] }
+      unless @fit
+        @implicit = !train_set.any? { |v| v[:rating] }
+      end
+
       unless @implicit
         ratings = train_set.map { |o| o[:rating] }
         check_ratings(ratings)
       end
 
       check_training_set(train_set)
-      create_maps(train_set)
+      update_maps(train_set)
 
       x_row = []
       x_col = []
       x_val = []
       value_key = @implicit ? :value : :rating
+      @rated = Hash.new { |hash, key| hash[key] = {} }
       train_set.each do |v|
-        x_row << @user_map[v[:user_id]]
-        x_col << @item_map[v[:item_id]]
+        u = @user_map[v[:user_id]]
+        i = @item_map[v[:item_id]]
+        @rated[u][i] = true
+
+        x_row << u
+        x_col << i
         x_val << (v[value_key] || 1)
       end
+      @rated.default = nil
 
       @m = @user_map.size
       @n = @item_map.size
@@ -46,20 +216,20 @@ module Cmfrec
       x_full = nil
       weight = nil
       lam_unique = nil
+      l1_lambda = 0
+      l1_lam_unique = nil
 
       uu = nil
       ii = nil
 
-      @user_info_map = {}
+      # side info
       u_row, u_col, u_sp, nnz_u, @m_u, p_ = process_info(user_info, @user_map, @user_info_map, :user_id)
-
-      @item_info_map = {}
       i_row, i_col, i_sp, nnz_i, @n_i, q = process_info(item_info, @item_map, @item_info_map, :item_id)
 
       @precompute_for_predictions = false
 
       # initialize w/ normal distribution
-      reset_values = true
+      reset_values = !@fit
 
       @a = Fiddle::Pointer.malloc([@m, @m_u].max * (@k_user + @k + @k_main) * Fiddle::SIZEOF_DOUBLE)
       @b = Fiddle::Pointer.malloc([@n, @n_i].max * (@k_item + @k + @k_main) * Fiddle::SIZEOF_DOUBLE)
@@ -73,16 +243,7 @@ module Cmfrec
       i_colmeans = Fiddle::Pointer.malloc(q * Fiddle::SIZEOF_DOUBLE)
 
       if @implicit
-        @w_main_multiplier = 1.0
-        @alpha = 1.0
-        @adjust_weight = false # downweight?
-        @apply_log_transf = false
-
-        # different defaults
-        @lambda_ = 1e0
-        @w_user = 10
-        @w_item = 10
-        @finalize_chol = false
+        set_implicit_vars
 
         args = [
           @a, @b,
@@ -92,6 +253,7 @@ module Cmfrec
           @m, @n, @k,
           x_row, x_col, x, nnz,
           @lambda_, lam_unique,
+          l1_lambda, l1_lam_unique,
           uu, @m_u, p_,
           ii, @n_i, q,
           u_row, u_col, u_sp, nnz_u,
@@ -125,6 +287,10 @@ module Cmfrec
 
         glob_mean = Fiddle::Pointer.malloc(Fiddle::SIZEOF_DOUBLE)
 
+        center = true
+        scale_lam = false
+        scale_lam_sideinfo = false
+
         args = [
           @bias_a, @bias_b,
           @a, @b,
@@ -138,8 +304,10 @@ module Cmfrec
           x_row, x_col, x, nnz,
           x_full,
           weight,
-          @user_bias, @item_bias,
+          @user_bias, @item_bias, center,
           @lambda_, lam_unique,
+          l1_lambda, l1_lam_unique,
+          scale_lam, scale_lam_sideinfo,
           uu, @m_u, p_,
           ii, @n_i, q,
           u_row, u_col, u_sp, nnz_u,
@@ -155,6 +323,7 @@ module Cmfrec
           nil, #B_plus_bias,
           nil, #precomputedBtB,
           nil, #precomputedTransBtBinvBt,
+          nil, #precomputedBtXbias
           nil, #precomputedBeTBeChol,
           nil, #precomputedBiTBi,
           nil, #precomputedTransCtCinvCt,
@@ -165,109 +334,13 @@ module Cmfrec
         @global_mean = real_array(glob_mean).first
       end
 
-      @u_colmeans = real_array(u_colmeans)
-      @i_colmeans = real_array(i_colmeans)
-      @u_colmeans_ptr = u_colmeans
-
-      self
-    end
-
-    def predict(data)
-      check_fit
-
-      data = to_dataset(data)
-
-      u = data.map { |v| @user_map[v[:user_id]] || -1 }
-      i = data.map { |v| @item_map[v[:item_id]] || -1 }
-
-      pred_a = int_ptr(u)
-      pred_b = int_ptr(i)
-      nnz = data.size
-      outp = Fiddle::Pointer.malloc(nnz * Fiddle::SIZEOF_DOUBLE)
-
-      FFI.predict_multiple(
-        @a, @k_user,
-        @b, @k_item,
-        @bias_a, @bias_b,
-        @global_mean,
-        @k, @k_main,
-        @m, @n,
-        pred_a, pred_b, nnz,
-        outp,
-        @nthreads
-      )
-
-      predictions = real_array(outp)
-
-      nan_index = predictions.each_index.select { |j| predictions[j].nan? }
-      if nan_index.any?
-        # TODO improve performance
-        user_bias = send(:user_bias)
-        item_bias = send(:item_bias)
-        nan_index.each do |j|
-          v = @global_mean
-          v += user_bias[u[j]] if user_bias && u[j] != -1
-          v += item_bias[i[j]] if item_bias && i[j] != -1
-          predictions[j] = v
-        end
-      end
-
-      predictions
-    end
-
-    def user_recs(user_id, count: 5, item_ids: nil)
-      check_fit
-      user = @user_map[user_id]
-
-      if user
-        if item_ids
-          # remove missing ids
-          item_ids = item_ids.select { |v| @item_map[v] }
-
-          data = item_ids.map { |v| {user_id: user_id, item_id: v} }
-          scores = predict(data)
-
-          item_ids.zip(scores).map do |item_id, score|
-            {item_id: item_id, score: score}
-          end
-        else
-          a_vec = @a[user * @k * Fiddle::SIZEOF_DOUBLE, @k * Fiddle::SIZEOF_DOUBLE]
-          a_bias = @bias_a ? @bias_a[user * Fiddle::SIZEOF_DOUBLE, Fiddle::SIZEOF_DOUBLE].unpack1("d") : 0
-          top_n(a_vec: a_vec, a_bias: a_bias, count: count)
-        end
-      else
-        # no items if user is unknown
-        # TODO maybe most popular items
-        []
-      end
-    end
-
-    # TODO add item_ids
-    def new_user_recs(data, count: 5, user_info: nil)
-      check_fit
-
-      a_vec, a_bias = factors_warm(data, user_info: user_info)
-      top_n(a_vec: a_vec, a_bias: a_bias, count: count)
-    end
-
-    def user_factors
-      read_factors(@a, [@m, @m_u].max, @k_user + @k + @k_main)
-    end
-
-    def item_factors
-      read_factors(@b, [@n, @n_i].max, @k_item + @k + @k_main)
-    end
+      @u_colmeans = u_colmeans
 
-    def user_bias
-      read_bias(@bias_a) if @bias_a
-    end
+      @fit = true
 
-    def item_bias
-      read_bias(@bias_b) if @bias_b
+      self
     end
 
-    private
-
     def set_params(
       k: 40, lambda_: 1e+1, method: "als", use_cg: true, user_bias: true,
       item_bias: true, add_implicit_features: false,
@@ -324,15 +397,14 @@ module Cmfrec
       @nthreads = nthreads
     end
 
-    def create_maps(train_set)
-      user_ids = train_set.map { |v| v[:user_id] }.uniq.sort
-      item_ids = train_set.map { |v| v[:item_id] }.uniq.sort
+    def update_maps(train_set)
+      raise ArgumentError, "Missing user_id" if train_set.any? { |v| v[:user_id].nil? }
+      raise ArgumentError, "Missing item_id" if train_set.any? { |v| v[:item_id].nil? }
 
-      raise ArgumentError, "Missing user_id" if user_ids.any?(&:nil?)
-      raise ArgumentError, "Missing item_id" if item_ids.any?(&:nil?)
-
-      @user_map = user_ids.zip(user_ids.size.times).to_h
-      @item_map = item_ids.zip(item_ids.size.times).to_h
+      train_set.each do |v|
+        @user_map[v[:user_id]] ||= @user_map.size
+        @item_map[v[:item_id]] ||= @item_map.size
+      end
     end
 
     def check_ratings(ratings)
@@ -349,7 +421,7 @@ module Cmfrec
     end
 
     def check_fit
-      raise "Not fit" unless defined?(@implicit)
+      raise "Not fit" unless @fit
     end
 
     def to_dataset(dataset)
@@ -386,11 +458,22 @@ module Cmfrec
       real_array(ptr)
     end
 
-    def top_n(a_vec:, a_bias:, count:)
+    def top_n(a_vec:, a_bias:, count:, rated: nil)
       include_ix = nil
       n_include = 0
-      exclude_ix = nil
-      n_exclude = 0
+
+      if rated
+        # assumes rated is unique and all items are known
+        # calling code is responsible for this
+        exclude_ix = int_ptr(rated)
+        n_exclude = rated.size
+        remaining = @item_map.size - n_exclude
+        return [] if remaining == 0
+        count = remaining if remaining < count
+      else
+        exclude_ix = nil
+        n_exclude = 0
+      end
 
       outp_ix = Fiddle::Pointer.malloc(count * Fiddle::SIZEOF_INT)
       outp_score = Fiddle::Pointer.malloc(count * Fiddle::SIZEOF_DOUBLE)
@@ -420,6 +503,16 @@ module Cmfrec
       data = to_dataset(data)
       user_info = to_dataset(user_info) if user_info
 
+      # remove unknown items
+      data, unknown_data = data.partition { |d| @item_map[d[:item_id]] }
+
+      if unknown_data.any?
+        # TODO warn for unknown items?
+        # warn "[cmfrec] Unknown items: #{unknown_data.map { |d| d[:item_id] }.join(", ")}"
+      end
+
+      item_ids = data.map { |d| @item_map[d[:item_id]] }
+
       nnz = data.size
       a_vec = Fiddle::Pointer.malloc((@k_user + @k + @k_main) * Fiddle::SIZEOF_DOUBLE)
       bias_a = Fiddle::Pointer.malloc(Fiddle::SIZEOF_DOUBLE)
@@ -448,6 +541,8 @@ module Cmfrec
 
       weight = nil
       lam_unique = nil
+      l1_lambda = 0
+      l1_lam_unique = nil
       n_max = @n
 
       if data.any?
@@ -458,7 +553,7 @@ module Cmfrec
           check_ratings(ratings)
         end
         xa = real_ptr(ratings)
-        x_col = int_ptr(data.map { |d| d[:item_id] })
+        x_col = int_ptr(item_ids)
       else
         xa = nil
         x_col = nil
@@ -472,11 +567,11 @@ module Cmfrec
           u_vec_sp, u_vec_x_col, nnz_u_vec,
           @na_as_zero_user,
           @nonneg,
-          @u_colmeans_ptr,
+          @u_colmeans,
           @b, @n, @c,
           xa, x_col, nnz,
           @k, @k_user, @k_item, @k_main,
-          @lambda_, @alpha,
+          @lambda_, l1_lambda, @alpha,
           @w_main, @w_user, @w_main_multiplier,
           @apply_log_transf,
           nil, #BeTBe,
@@ -487,6 +582,9 @@ module Cmfrec
       else
         cb = nil
 
+        scale_lam = false
+        scale_lam_sideinfo = false
+
         args = [
           a_vec, bias_a,
           u_vec, p_,
@@ -495,17 +593,20 @@ module Cmfrec
           @na_as_zero_user, @na_as_zero,
           @nonneg,
           @c, cb,
-          @global_mean, @bias_b, @u_colmeans_ptr,
+          @global_mean, @bias_b, @u_colmeans,
           xa, x_col, nnz, xa_dense,
           @n, weight, @b, @bi,
           @add_implicit_features,
           @k, @k_user, @k_item, @k_main,
           @lambda_, lam_unique,
+          l1_lambda, l1_lam_unique,
+          scale_lam, scale_lam_sideinfo,
           @w_main, @w_user, @w_implicit,
           n_max,
           @include_all_x,
-          nil, #TransBtBinvBt,
           nil, #BtB,
+          nil, #TransBtBinvBt,
+          nil, #BtXbias,
           nil, #BeTBeChol,
           nil, #BiTBi,
           nil, #CtCw,
@@ -515,7 +616,7 @@ module Cmfrec
         check_status FFI.factors_collective_explicit_single(*fiddle_args(args))
       end
 
-      [a_vec, real_array(bias_a).first]
+      [a_vec, real_array(bias_a).first, item_ids.uniq]
     end
 
     # convert boolean to int
@@ -572,5 +673,126 @@ module Cmfrec
     def real_array(ptr)
       ptr.to_s(ptr.size).unpack("d*")
     end
+
+    def set_implicit_vars
+      @w_main_multiplier = 1.0
+      @alpha = 1.0
+      @adjust_weight = false # downweight?
+      @apply_log_transf = false
+
+      # different defaults
+      @lambda_ = 1e0
+      @w_user = 10
+      @w_item = 10
+      @finalize_chol = false
+    end
+
+    def dump_ptr(ptr)
+      ptr.to_s(ptr.size) if ptr
+    end
+
+    def load_ptr(str)
+      Fiddle::Pointer[str] if str
+    end
+
+    def marshal_dump
+      obj = {
+        implicit: @implicit
+      }
+
+      # options
+      obj[:factors] = @k
+      obj[:epochs] = @niter
+      obj[:verbose] = @verbose
+
+      # factors
+      obj[:user_map] = @user_map
+      obj[:item_map] = @item_map
+      obj[:rated] = @rated
+      obj[:user_factors] = dump_ptr(@a)
+      obj[:item_factors] = dump_ptr(@b)
+
+      # bias
+      obj[:user_bias] = dump_ptr(@bias_a)
+      obj[:item_bias] = dump_ptr(@bias_b)
+
+      # mean
+      obj[:global_mean] = @global_mean
+
+      # side info
+      obj[:user_info_map] = @user_info_map
+      obj[:item_info_map] = @item_info_map
+      obj[:user_info_factors] = dump_ptr(@c)
+      obj[:item_info_factors] = dump_ptr(@d)
+
+      # implicit features
+      obj[:add_implicit_features] = @add_implicit_features
+      obj[:user_factors_implicit] = dump_ptr(@ai)
+      obj[:item_factors_implicit] = dump_ptr(@bi)
+
+      unless @implicit
+        obj[:min_rating] = @min_rating
+        obj[:max_rating] = @max_rating
+      end
+
+      obj[:user_means] = dump_ptr(@u_colmeans)
+
+      obj
+    end
+
+    def marshal_load(obj)
+      @implicit = obj[:implicit]
+
+      # options
+      set_params(
+        k: obj[:factors],
+        niter: obj[:epochs],
+        verbose: obj[:verbose],
+        user_bias: !obj[:user_bias].nil?,
+        item_bias: !obj[:item_bias].nil?,
+        add_implicit_features: obj[:add_implicit_features]
+      )
+
+      # factors
+      @user_map = obj[:user_map]
+      @item_map = obj[:item_map]
+      @rated = obj[:rated] || {}
+      @a = load_ptr(obj[:user_factors])
+      @b = load_ptr(obj[:item_factors])
+
+      # bias
+      @bias_a = load_ptr(obj[:user_bias])
+      @bias_b = load_ptr(obj[:item_bias])
+
+      # mean
+      @global_mean = obj[:global_mean]
+
+      # side info
+      @user_info_map = obj[:user_info_map]
+      @item_info_map = obj[:item_info_map]
+      @c = load_ptr(obj[:user_info_factors])
+      @d = load_ptr(obj[:item_info_factors])
+
+      # implicit features
+      @add_implicit_features = obj[:add_implicit_features]
+      @ai = load_ptr(obj[:user_factors_implicit])
+      @bi = load_ptr(obj[:item_factors_implicit])
+
+      unless @implicit
+        @min_rating = obj[:min_rating]
+        @max_rating = obj[:max_rating]
+      end
+
+      @u_colmeans = load_ptr(obj[:user_means])
+
+      @m = @user_map.size
+      @n = @item_map.size
+      @m_u = @user_info_map.size
+      @n_i = @item_info_map.size
+
+      set_implicit_vars if @implicit
+
+      @fit = @m > 0
+    end
   end
 end

data/lib/cmfrec/version.rb

@@ -1,3 +1,3 @@
 module Cmfrec
-  VERSION = "0.1.1"
+  VERSION = "0.1.5"
 end

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: cmfrec
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.5
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-28 00:00:00.000000000 Z
+date: 2021-08-11 00:00:00.000000000 Z
 dependencies: []
 description:
-email: andrew@chartkick.com
+email: andrew@ankane.org
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -20,10 +20,12 @@ files:
 - LICENSE.txt
 - README.md
 - lib/cmfrec.rb
+- lib/cmfrec/data.rb
 - lib/cmfrec/ffi.rb
 - lib/cmfrec/recommender.rb
 - lib/cmfrec/version.rb
 - vendor/LICENSE.txt
+- vendor/libcmfrec.arm64.dylib
 - vendor/libcmfrec.dylib
 - vendor/libcmfrec.so
 homepage: https://github.com/ankane/cmfrec
@@ -45,7 +47,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: Recommendations for Ruby using collective matrix factorization