cmfrec 0.1.4 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bb7b07ae46500a545f1a130dfc5648aa3f925f9b5766a6c70a1652c7b5732182
-  data.tar.gz: e89a6d1900cda651dc6b0aac2899050e28680cddfb6b39b6b5eacfe467b59aad
+  metadata.gz: 8d16ab98cb7de22042eaf353a9d41d0b7a4214a631a373c553f73825418c026a
+  data.tar.gz: 9ab678a9d389b835b4dfd91d14c372d5acfef950bf068ac46d2d879af04f0fcc
 SHA512:
-  metadata.gz: 117aa6952fe0ab8ddebfaece6655cf479a7adbab7d6f634e7d3428c72824a410812c037ae006366180a9691a6d160d8065b777a9c10a33a5ccfefedb28c99ec6
-  data.tar.gz: 57985a055705b820226a2aa1451453383ee3509e43225f8fdb09e713c4530754b0b608f7d1b4814973b43e3d625f824f9f87939687d015b352cc8905f7b4f118
+  metadata.gz: 175d3c91056d2e8734af6961471c98be76e4d5f6b85faaecdfd3b39a220efafa70150983e9d74efb1a1211a29e6c867d6b1d7f482cc34c55500268d29b40158c
+  data.tar.gz: faaed621391ccc7d94f2e6309481b24f7db62fbe86eb6c1bbb35445dde0cabf32c41791af4ebcd25e530bdc1f6319f1b4477b6c4d6ca1bf77353d3a0c4ae8d5c

data/CHANGELOG.md

@@ -1,3 +1,24 @@
+## 0.1.7 (2022-03-22)
+
+- Improved ARM detection
+- Fixed error with `load_movielens`
+- Fixed duplicates in `item_info` with `load_movielens`
+
+## 0.1.6 (2021-08-12)
+
+- Added `user_ids` and `item_ids` methods
+- Added `user_id` argument to `user_factors`
+- Added `item_id` argument to `item_factors`
+- Added `user_id` argument to `user_bias`
+- Added `item_id` argument to `item_bias`
+- Added `item_ids` argument to `new_user_recs`
+- Fixed order for `user_recs`
+
+## 0.1.5 (2021-08-10)
+
+- Fixed issue with `user_recs` and `new_user_recs` returning rated items
+- Fixed error with `new_user_recs`
+
 ## 0.1.4 (2021-02-04)
 
 - Added support for saving and loading recommenders

data/README.md

@@ -1,4 +1,4 @@
-# cmfrec
+# cmfrec Ruby
 
 :fire: Recommendations for Ruby, powered by [cmfrec](https://github.com/david-cortes/cmfrec)
 
@@ -6,7 +6,7 @@
 - Works with explicit and implicit feedback
 - Uses high-performance matrix factorization
 
-[![Build Status](https://github.com/ankane/cmfrec/workflows/build/badge.svg?branch=master)](https://github.com/ankane/cmfrec/actions)
+[![Build Status](https://github.com/ankane/cmfrec-ruby/workflows/build/badge.svg?branch=master)](https://github.com/ankane/cmfrec-ruby/actions)
 
 ## Installation
 
@@ -58,8 +58,8 @@ Get recommendations for a new user
 
 ```ruby
 recommender.new_user_recs([
-  {item_id: 1, value: 5},
-  {item_id: 2, value: 3}
+  {item_id: 1, rating: 5},
+  {item_id: 2, rating: 3}
 ])
 ```
 
@@ -150,11 +150,7 @@ recommender.predict(ratings.last(20000))
 [Ahoy](https://github.com/ankane/ahoy) is a great source for implicit feedback
 
 ```ruby
-views = Ahoy::Event.
-  where(name: "Viewed post").
-  group(:user_id).
-  group("properties->>'post_id'"). # postgres syntax
-  count
+views = Ahoy::Event.where(name: "Viewed post").group(:user_id).group_prop(:post_id).count
 
 data =
   views.map do |(user_id, post_id), count|
@@ -230,8 +226,17 @@ bin = File.binread("recommender.bin")
 recommender = Marshal.load(bin)
 ```
 
+Alternatively, you can store only the factors and use a library like [Neighbor](https://github.com/ankane/neighbor). See the [examples](https://github.com/ankane/neighbor/tree/master/examples) for Disco, which has a similar API. For explicit feedback, you should [disable the bias](#explicit-feedback) with this approach.
+
 ## Reference
 
+Get ids
+
+```ruby
+recommender.user_ids
+recommender.item_ids
+```
+
 Get the global mean
 
 ```ruby
@@ -262,22 +267,22 @@ Cmfrec.ffi_lib = "path/to/cmfrec.dll"
 
 ## History
 
-View the [changelog](https://github.com/ankane/cmfrec/blob/master/CHANGELOG.md)
+View the [changelog](https://github.com/ankane/cmfrec-ruby/blob/master/CHANGELOG.md)
 
 ## Contributing
 
 Everyone is encouraged to help improve this project. Here are a few ways you can help:
 
-- [Report bugs](https://github.com/ankane/cmfrec/issues)
-- Fix bugs and [submit pull requests](https://github.com/ankane/cmfrec/pulls)
+- [Report bugs](https://github.com/ankane/cmfrec-ruby/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/cmfrec-ruby/pulls)
 - Write, clarify, or fix documentation
 - Suggest or add new features
 
 To get started with development:
 
 ```sh
-git clone https://github.com/ankane/cmfrec.git
-cd cmfrec
+git clone https://github.com/ankane/cmfrec-ruby.git
+cd cmfrec-ruby
 bundle install
 bundle exec rake vendor:all
 bundle exec rake test

data/lib/cmfrec/data.rb

@@ -3,11 +3,11 @@ module Cmfrec
     def load_movielens
       require "csv"
 
-      data_path = download_file("ml-100k/u.data", "http://files.grouplens.org/datasets/movielens/ml-100k/u.data",
+      data_path = download_file("ml-100k/u.data", "https://files.grouplens.org/datasets/movielens/ml-100k/u.data",
         file_hash: "06416e597f82b7342361e41163890c81036900f418ad91315590814211dca490")
-      user_path = download_file("ml-100k/u.user", "http://files.grouplens.org/datasets/movielens/ml-100k/u.user",
+      user_path = download_file("ml-100k/u.user", "https://files.grouplens.org/datasets/movielens/ml-100k/u.user",
         file_hash: "f120e114da2e8cf314fd28f99417c94ae9ddf1cb6db8ce0e4b5995d40e90e62c")
-      item_path = download_file("ml-100k/u.item", "http://files.grouplens.org/datasets/movielens/ml-100k/u.item",
+      item_path = download_file("ml-100k/u.item", "https://files.grouplens.org/datasets/movielens/ml-100k/u.item",
         file_hash: "553841ebc7de3a0fd0d6b62a204ea30c1e651aacfb2814c7a6584ac52f2c5701")
 
       # convert u.item to utf-8
@@ -24,8 +24,13 @@ module Cmfrec
 
       item_info = []
       movies = {}
+      movie_names = {}
       genres = %w(unknown action adventure animation childrens comedy crime documentary drama fantasy filmnoir horror musical mystery romance scifi thriller war western)
       CSV.parse(movies_str, col_sep: "|", converters: [:numeric]) do |row|
+        # filter duplicates
+        next if movie_names[row[1]]
+        movie_names[row[1]] = true
+
         movies[row[0]] = row[1]
         item = {item_id: row[1], year: row[2] ? Date.parse(row[2]).year : 1970}
         genres.each_with_index do |genre, i|
@@ -49,7 +54,10 @@ module Cmfrec
     private
 
     def download_file(fname, origin, file_hash:)
+      require "digest"
       require "fileutils"
+      require "net/http"
+      require "tmpdir"
 
       # TODO handle this better
       raise "No HOME" unless ENV["HOME"]
@@ -58,10 +66,6 @@ module Cmfrec
 
       return dest if File.exist?(dest)
 
-      require "digest"
-      require "net/http"
-      require "tmpdir"
-
       temp_path = "#{Dir.tmpdir}/cmfrec-#{Time.now.to_f}" # TODO better name
 
       digest = Digest::SHA2.new

data/lib/cmfrec/recommender.rb

@@ -67,21 +67,10 @@ module Cmfrec
       user = @user_map[user_id]
 
       if user
-        if item_ids
-          # remove missing ids
-          item_ids = item_ids.select { |v| @item_map[v] }
-
-          data = item_ids.map { |v| {user_id: user_id, item_id: v} }
-          scores = predict(data)
-
-          item_ids.zip(scores).map do |item_id, score|
-            {item_id: item_id, score: score}
-          end
-        else
-          a_vec = @a[user * @k * Fiddle::SIZEOF_DOUBLE, @k * Fiddle::SIZEOF_DOUBLE]
-          a_bias = @bias_a ? @bias_a[user * Fiddle::SIZEOF_DOUBLE, Fiddle::SIZEOF_DOUBLE].unpack1("d") : 0
-          top_n(a_vec: a_vec, a_bias: a_bias, count: count)
-        end
+        a_vec = @a[user * @k * Fiddle::SIZEOF_DOUBLE, @k * Fiddle::SIZEOF_DOUBLE]
+        a_bias = @bias_a ? @bias_a[user * Fiddle::SIZEOF_DOUBLE, Fiddle::SIZEOF_DOUBLE].unpack1("d") : 0
+        # @rated[user] will be nil for recommenders saved before 0.1.5
+        top_n(a_vec: a_vec, a_bias: a_bias, count: count, rated: (@rated[user] || {}).keys, item_ids: item_ids)
       else
         # no items if user is unknown
         # TODO maybe most popular items
@@ -89,28 +78,35 @@ module Cmfrec
       end
     end
 
-    # TODO add item_ids
-    def new_user_recs(data, count: 5, user_info: nil)
+    def new_user_recs(data, count: 5, user_info: nil, item_ids: nil)
       check_fit
 
-      a_vec, a_bias = factors_warm(data, user_info: user_info)
-      top_n(a_vec: a_vec, a_bias: a_bias, count: count)
+      a_vec, a_bias, rated = factors_warm(data, user_info: user_info)
+      top_n(a_vec: a_vec, a_bias: a_bias, count: count, rated: rated, item_ids: item_ids)
+    end
+
+    def user_ids
+      @user_map.keys
     end
 
-    def user_factors
-      read_factors(@a, [@m, @m_u].max, @k_user + @k + @k_main)
+    def item_ids
+      @item_map.keys
     end
 
-    def item_factors
-      read_factors(@b, [@n, @n_i].max, @k_item + @k + @k_main)
+    def user_factors(user_id = nil)
+      read_factors(@a, [@m, @m_u].max, @k_user + @k + @k_main, user_id, @user_map)
     end
 
-    def user_bias
-      read_bias(@bias_a) if @bias_a
+    def item_factors(item_id = nil)
+      read_factors(@b, [@n, @n_i].max, @k_item + @k + @k_main, item_id, @item_map)
     end
 
-    def item_bias
-      read_bias(@bias_b) if @bias_b
+    def user_bias(user_id = nil)
+      read_bias(@bias_a, user_id, @user_map) if @bias_a
+    end
+
+    def item_bias(item_id = nil)
+      read_bias(@bias_b, item_id, @item_map) if @bias_b
     end
 
     def similar_items(item_id, count: 5)
@@ -191,11 +187,17 @@ module Cmfrec
       x_col = []
       x_val = []
       value_key = @implicit ? :value : :rating
+      @rated = Hash.new { |hash, key| hash[key] = {} }
       train_set.each do |v|
-        x_row << @user_map[v[:user_id]]
-        x_col << @item_map[v[:item_id]]
+        u = @user_map[v[:user_id]]
+        i = @item_map[v[:item_id]]
+        @rated[u][i] = true
+
+        x_row << u
+        x_col << i
         x_val << (v[value_key] || 1)
       end
+      @rated.default = nil
 
       @m = @user_map.size
       @n = @item_map.size
@@ -435,26 +437,59 @@ module Cmfrec
       end
     end
 
-    def read_factors(ptr, d1, d2)
-      arr = []
-      offset = 0
+    def read_factors(ptr, d1, d2, id, map)
       width = d2 * Fiddle::SIZEOF_DOUBLE
-      d1.times do |i|
-        arr << ptr[offset, width].unpack("d*")
-        offset += width
+      if id
+        i = map[id]
+        ptr[i * width, width].unpack("d*") if i
+      else
+        arr = []
+        offset = 0
+        d1.times do |i|
+          arr << ptr[offset, width].unpack("d*")
+          offset += width
+        end
+        arr
       end
-      arr
     end
 
-    def read_bias(ptr)
-      real_array(ptr)
+    def read_bias(ptr, id, map)
+      if id
+        i = map[id]
+        ptr[i * Fiddle::SIZEOF_DOUBLE, Fiddle::SIZEOF_DOUBLE].unpack1("d") if i
+      else
+        real_array(ptr)
+      end
     end
 
-    def top_n(a_vec:, a_bias:, count:)
-      include_ix = nil
-      n_include = 0
-      exclude_ix = nil
-      n_exclude = 0
+    def top_n(a_vec:, a_bias:, count:, rated: nil, item_ids: nil)
+      if item_ids
+        # remove missing ids
+        item_ids = item_ids.map { |v| @item_map[v] }.compact
+        return [] if item_ids.empty?
+
+        include_ix = int_ptr(item_ids)
+        n_include = item_ids.size
+
+        # TODO uncomment in 0.2.0
+        count = n_include # if n_include < count
+      else
+        include_ix = nil
+        n_include = 0
+      end
+
+      if rated && !item_ids
+        # assumes rated is unique and all items are known
+        # calling code is responsible for this
+        exclude_ix = int_ptr(rated)
+        n_exclude = rated.size
+        remaining = @item_map.size - n_exclude
+        return [] if remaining == 0
+        count = remaining if remaining < count
+      else
+        exclude_ix = nil
+        n_exclude = 0
+      end
 
       outp_ix = Fiddle::Pointer.malloc(count * Fiddle::SIZEOF_INT)
       outp_score = Fiddle::Pointer.malloc(count * Fiddle::SIZEOF_DOUBLE)
@@ -484,6 +519,16 @@ module Cmfrec
       data = to_dataset(data)
       user_info = to_dataset(user_info) if user_info
 
+      # remove unknown items
+      data, unknown_data = data.partition { |d| @item_map[d[:item_id]] }
+
+      if unknown_data.any?
+        # TODO warn for unknown items?
+        # warn "[cmfrec] Unknown items: #{unknown_data.map { |d| d[:item_id] }.join(", ")}"
+      end
+
+      item_ids = data.map { |d| @item_map[d[:item_id]] }
+
       nnz = data.size
       a_vec = Fiddle::Pointer.malloc((@k_user + @k + @k_main) * Fiddle::SIZEOF_DOUBLE)
       bias_a = Fiddle::Pointer.malloc(Fiddle::SIZEOF_DOUBLE)
@@ -524,7 +569,7 @@ module Cmfrec
           check_ratings(ratings)
         end
         xa = real_ptr(ratings)
-        x_col = int_ptr(data.map { |d| d[:item_id] })
+        x_col = int_ptr(item_ids)
       else
         xa = nil
         x_col = nil
@@ -587,7 +632,7 @@ module Cmfrec
         check_status FFI.factors_collective_explicit_single(*fiddle_args(args))
       end
 
-      [a_vec, real_array(bias_a).first]
+      [a_vec, real_array(bias_a).first, item_ids.uniq]
     end
 
     # convert boolean to int
@@ -679,6 +724,7 @@ module Cmfrec
       # factors
       obj[:user_map] = @user_map
       obj[:item_map] = @item_map
+      obj[:rated] = @rated
       obj[:user_factors] = dump_ptr(@a)
       obj[:item_factors] = dump_ptr(@b)
 
@@ -726,6 +772,7 @@ module Cmfrec
       # factors
       @user_map = obj[:user_map]
       @item_map = obj[:item_map]
+      @rated = obj[:rated] || {}
       @a = load_ptr(obj[:user_factors])
       @b = load_ptr(obj[:item_factors])
 

data/lib/cmfrec/version.rb

@@ -1,3 +1,3 @@
 module Cmfrec
-  VERSION = "0.1.4"
+  VERSION = "0.1.7"
 end

data/lib/cmfrec.rb

@@ -19,7 +19,7 @@ module Cmfrec
     if Gem.win_platform?
       "cmfrec.dll"
     elsif RbConfig::CONFIG["host_os"] =~ /darwin/i
-      if RbConfig::CONFIG["host_cpu"] =~ /arm/i
+      if RbConfig::CONFIG["host_cpu"] =~ /arm|aarch64/i
         "libcmfrec.arm64.dylib"
       else
         "libcmfrec.dylib"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cmfrec
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.7
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-05 00:00:00.000000000 Z
+date: 2022-03-22 00:00:00.000000000 Z
 dependencies: []
 description:
 email: andrew@ankane.org
@@ -28,7 +28,7 @@ files:
 - vendor/libcmfrec.arm64.dylib
 - vendor/libcmfrec.dylib
 - vendor/libcmfrec.so
-homepage: https://github.com/ankane/cmfrec
+homepage: https://github.com/ankane/cmfrec-ruby
 licenses:
 - MIT
 metadata: {}
@@ -47,7 +47,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Recommendations for Ruby using collective matrix factorization