cmfrec 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6dbbc801e415a4f505ffc436be23ccf066d144da072669e782b88c02e14b0f8
-  data.tar.gz: 3851230f0a4dc4be9fbc24fe81681de0758bdbb583803780f8e07b10741f4bd1
+  metadata.gz: d3cc7d70530eefc7b13324753b454d03573da7d832c46cb4dee2ab9213eafcdd
+  data.tar.gz: 662af2ec4ab1a1bd33a39c18773d0c48967422286a2fa137960772b7e67d437a
 SHA512:
-  metadata.gz: 103c09a7c0e13fca3cb81dc68c667e776fc965a485cf358e0fc8f350a97474b54bdfc0910f4472051020dbe34b8c097908c1024d5fc036ad77d0444372885109
-  data.tar.gz: b107b36333f714106d981168f24fda48a2a211f288a2dbe01f570adb607b7d6a5215c18df4c67d159bb367d652f3d10faeb4dbc66688eaf117c5b7a1432ee951
+  metadata.gz: 8fd6f1f8f0bd7d7c870c28fb57a0cec89aacf2d27aed53b5d68fb6935f5071dbe73931a5ff776f4a864f0cc91a17c793eabfe2a2b21f9b368a4c36ada5cb929d
+  data.tar.gz: 116d26ddafeeb439ef0895e30805afa0d2d2a453aeb369cf7122f13f5bf3ad457dac65c974d26debc84f353baf5f0c889c559e56620ef7458af4968ee9f5262a

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.3.0 (2024-10-23)
+
+- Changed dataset directory to match XDG Base Directory Specification
+- Removed dependency on `csv` gem for `load_movielens`
+- Dropped support for marshal serialization
+- Dropped support for Ruby < 3.1
+
+## 0.2.1 (2022-07-11)
+
+- Added support for JSON serialization
+
 ## 0.2.0 (2022-06-14)
 
 - Updated cmfrec to 3.4.2

data/LICENSE.txt

@@ -1,7 +1,7 @@
 MIT License
 
 Copyright (c) 2020 David Cortes
-Copyright (c) 2020-2021 Andrew Kane
+Copyright (c) 2020-2024 Andrew Kane
 
 All rights reserved.
 

data/README.md

@@ -6,7 +6,7 @@
 - Works with explicit and implicit feedback
 - Uses high-performance matrix factorization
 
-[![Build Status](https://github.com/ankane/cmfrec-ruby/workflows/build/badge.svg?branch=master)](https://github.com/ankane/cmfrec-ruby/actions)
+[![Build Status](https://github.com/ankane/cmfrec-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/cmfrec-ruby/actions)
 
 ## Installation
 
@@ -82,11 +82,11 @@ Add side information about users, items, or both
 ```ruby
 user_info = [
   {user_id: 1, cats: 1, dogs: 0},
-  {user_id: 2, cats: 2, dogs: 1},
+  {user_id: 2, cats: 2, dogs: 1}
 ]
 item_info = [
   {item_id: 1, genre_comedy: 1, genre_drama: 0},
-  {item_id: 2, genre_comedy: 0, genre_drama: 1},
+  {item_id: 2, genre_comedy: 0, genre_drama: 1}
 ]
 recommender.fit(ratings, user_info: user_info, item_info: item_info)
 ```
@@ -213,17 +213,17 @@ Rover.read_csv("ratings.csv")
 Store the recommender
 
 ```ruby
-bin = Marshal.dump(recommender)
-File.binwrite("recommender.bin", bin)
+json = recommender.to_json
+File.write("recommender.json", json)
 ```
 
-> You can save it to a file, database, or any other storage system
+The serialized recommender includes user activity from the training data (to avoid recommending previously rated items), so be sure to protect it. You can save it to a file, database, or any other storage system, or use a tool like [Trove](https://github.com/ankane/trove). Also, user and item IDs should be integers or strings for this.
 
 Load a recommender
 
 ```ruby
-bin = File.binread("recommender.bin")
-recommender = Marshal.load(bin)
+json = File.read("recommender.json")
+recommender = Cmfrec::Recommender.load_json(json)
 ```
 
 Alternatively, you can store only the factors and use a library like [Neighbor](https://github.com/ankane/neighbor). See the [examples](https://github.com/ankane/neighbor/tree/master/examples) for Disco, which has a similar API. For explicit feedback, you should [disable the bias](#explicit-feedback) with this approach.

data/lib/cmfrec/data.rb

@@ -1,8 +1,6 @@
 module Cmfrec
   module Data
     def load_movielens
-      require "csv"
-
       data_path = download_file("ml-100k/u.data", "https://files.grouplens.org/datasets/movielens/ml-100k/u.data",
         file_hash: "06416e597f82b7342361e41163890c81036900f418ad91315590814211dca490")
       user_path = download_file("ml-100k/u.user", "https://files.grouplens.org/datasets/movielens/ml-100k/u.user",
@@ -10,11 +8,9 @@ module Cmfrec
       item_path = download_file("ml-100k/u.item", "https://files.grouplens.org/datasets/movielens/ml-100k/u.item",
         file_hash: "553841ebc7de3a0fd0d6b62a204ea30c1e651aacfb2814c7a6584ac52f2c5701")
 
-      # convert u.item to utf-8
-      movies_str = File.read(item_path).encode("UTF-8", "binary", invalid: :replace, undef: :replace, replace: "")
-
       user_info = []
-      CSV.foreach(user_path, col_sep: "|") do |row|
+      File.foreach(user_path) do |line|
+        row = line.split("|")
         user = {user_id: row[0].to_i}
         10.times do |i|
           user[:"region#{i}"] = row[4][0] == i.to_s ? 1 : 0
@@ -26,26 +22,28 @@ module Cmfrec
       movies = {}
       movie_names = {}
       genres = %w(unknown action adventure animation childrens comedy crime documentary drama fantasy filmnoir horror musical mystery romance scifi thriller war western)
-      CSV.parse(movies_str, col_sep: "|", converters: [:numeric]) do |row|
+      File.foreach(item_path) do |line|
+        row = line.encode("UTF-8", "ISO-8859-1").split("|")
         movies[row[0]] = row[1]
 
         # filter duplicates
         next if movie_names[row[1]]
         movie_names[row[1]] = true
 
-        item = {item_id: row[1], year: row[2] ? Date.parse(row[2]).year : 1970}
+        item = {item_id: row[1], year: !row[2].empty? ? Date.parse(row[2]).year : 1970}
         genres.each_with_index do |genre, i|
-          item[:"genre_#{genre}"] = row[i + 5]
+          item[:"genre_#{genre}"] = row[i + 5].to_i
         end
         item_info << item
       end
 
       data = []
-      CSV.foreach(data_path, col_sep: "\t", converters: [:numeric]) do |row|
+      File.foreach(data_path) do |line|
+        row = line.split("\t")
         data << {
-          user_id: row[0],
+          user_id: row[0].to_i,
           item_id: movies[row[1]],
-          rating: row[2]
+          rating: row[2].to_i
         }
       end
 
@@ -60,9 +58,8 @@ module Cmfrec
       require "net/http"
       require "tmpdir"
 
-      # TODO handle this better
-      raise "No HOME" unless ENV["HOME"]
-      dest = "#{ENV["HOME"]}/.cmfrec/#{fname}"
+      cache_home = ENV["XDG_CACHE_HOME"] || "#{ENV.fetch("HOME")}/.cache"
+      dest = "#{cache_home}/cmfrec/#{fname}"
       FileUtils.mkdir_p(File.dirname(dest))
 
       return dest if File.exist?(dest)

data/lib/cmfrec/recommender.rb

@@ -249,6 +249,68 @@ module Cmfrec
       similar(user_id, @user_map, user_factors, count, user_index)
     end
 
+    def to_json
+      require "base64"
+      require "json"
+
+      obj = {
+        implicit: @implicit
+      }
+
+      # options
+      obj[:factors] = @k
+      obj[:epochs] = @niter
+      obj[:verbose] = @verbose
+
+      # factors
+      obj[:user_ids] = @user_map.keys
+      obj[:item_ids] = @item_map.keys
+      obj[:rated] = @user_map.map { |_, u| (@rated[u] || {}).keys }
+      obj[:user_factors] = json_dump_ptr(@a)
+      obj[:item_factors] = json_dump_ptr(@b)
+
+      # bias
+      obj[:user_bias] = json_dump_ptr(@bias_a)
+      obj[:item_bias] = json_dump_ptr(@bias_b)
+
+      # mean
+      obj[:global_mean] = @global_mean
+
+      unless (@user_info_map.keys + @item_info_map.keys).all? { |v| v.is_a?(Symbol) }
+        raise "Side info keys must be symbols to save"
+      end
+
+      # side info
+      obj[:user_info_ids] = @user_info_map.keys
+      obj[:item_info_ids] = @item_info_map.keys
+      obj[:user_info_factors] = json_dump_ptr(@c)
+      obj[:item_info_factors] = json_dump_ptr(@d)
+
+      # implicit features
+      obj[:add_implicit_features] = @add_implicit_features
+      obj[:user_factors_implicit] = json_dump_ptr(@ai)
+      obj[:item_factors_implicit] = json_dump_ptr(@bi)
+
+      unless @implicit
+        obj[:min_rating] = @min_rating
+        obj[:max_rating] = @max_rating
+      end
+
+      obj[:user_means] = json_dump_ptr(@u_colmeans)
+
+      JSON.generate(obj)
+    end
+
+    def self.load_json(json)
+      require "json"
+
+      obj = JSON.parse(json)
+
+      recommender = new
+      recommender.send(:json_load, obj)
+      recommender
+    end
+
     private
 
     def user_index
@@ -452,7 +514,7 @@ module Cmfrec
           nil, #precomputedBiTBi,
           nil, #precomputedTransCtCinvCt,
           nil, #precomputedCtCw
-          nil, #precomputedCtUbias
+          nil  #precomputedCtUbias
         ]
         check_status FFI.fit_collective_explicit_als(*fiddle_args(args))
 
@@ -749,103 +811,60 @@ module Cmfrec
       @finalize_chol = false
     end
 
-    def dump_ptr(ptr)
-      ptr.to_s(ptr.size) if ptr
+    def json_dump_ptr(ptr)
+      Base64.strict_encode64(ptr.to_s(ptr.size)) if ptr
     end
 
-    def load_ptr(str)
-      Fiddle::Pointer[str] if str
+    def json_load_ptr(str)
+      Fiddle::Pointer[Base64.strict_decode64(str)] if str
     end
 
-    def marshal_dump
-      obj = {
-        implicit: @implicit
-      }
-
-      # options
-      obj[:factors] = @k
-      obj[:epochs] = @niter
-      obj[:verbose] = @verbose
-
-      # factors
-      obj[:user_map] = @user_map
-      obj[:item_map] = @item_map
-      obj[:rated] = @rated
-      obj[:user_factors] = dump_ptr(@a)
-      obj[:item_factors] = dump_ptr(@b)
-
-      # bias
-      obj[:user_bias] = dump_ptr(@bias_a)
-      obj[:item_bias] = dump_ptr(@bias_b)
-
-      # mean
-      obj[:global_mean] = @global_mean
-
-      # side info
-      obj[:user_info_map] = @user_info_map
-      obj[:item_info_map] = @item_info_map
-      obj[:user_info_factors] = dump_ptr(@c)
-      obj[:item_info_factors] = dump_ptr(@d)
-
-      # implicit features
-      obj[:add_implicit_features] = @add_implicit_features
-      obj[:user_factors_implicit] = dump_ptr(@ai)
-      obj[:item_factors_implicit] = dump_ptr(@bi)
-
-      unless @implicit
-        obj[:min_rating] = @min_rating
-        obj[:max_rating] = @max_rating
-      end
-
-      obj[:user_means] = dump_ptr(@u_colmeans)
-
-      obj
-    end
+    def json_load(obj)
+      require "base64"
 
-    def marshal_load(obj)
-      @implicit = obj[:implicit]
+      @implicit = obj["implicit"]
 
       # options
       set_params(
-        k: obj[:factors],
-        niter: obj[:epochs],
-        verbose: obj[:verbose],
-        user_bias: !obj[:user_bias].nil?,
-        item_bias: !obj[:item_bias].nil?,
-        add_implicit_features: obj[:add_implicit_features]
+        k: obj["factors"],
+        niter: obj["epochs"],
+        verbose: obj["verbose"],
+        user_bias: !obj["user_bias"].nil?,
+        item_bias: !obj["item_bias"].nil?,
+        add_implicit_features: obj["add_implicit_features"]
       )
 
       # factors
-      @user_map = obj[:user_map]
-      @item_map = obj[:item_map]
-      @rated = obj[:rated] || {}
-      @a = load_ptr(obj[:user_factors])
-      @b = load_ptr(obj[:item_factors])
+      @user_map = obj["user_ids"].map.with_index.to_h
+      @item_map = obj["item_ids"].map.with_index.to_h
+      @rated = obj["rated"].map.with_index.to_h { |r, i| [i, r.to_h { |v| [v, true] }] }
+      @a = json_load_ptr(obj["user_factors"])
+      @b = json_load_ptr(obj["item_factors"])
 
       # bias
-      @bias_a = load_ptr(obj[:user_bias])
-      @bias_b = load_ptr(obj[:item_bias])
+      @bias_a = json_load_ptr(obj["user_bias"])
+      @bias_b = json_load_ptr(obj["item_bias"])
 
       # mean
-      @global_mean = obj[:global_mean]
+      @global_mean = obj["global_mean"]
 
       # side info
-      @user_info_map = obj[:user_info_map]
-      @item_info_map = obj[:item_info_map]
-      @c = load_ptr(obj[:user_info_factors])
-      @d = load_ptr(obj[:item_info_factors])
+      @user_info_map = obj["user_info_ids"].map(&:to_sym).map.with_index.to_h
+      @item_info_map = obj["item_info_ids"].map(&:to_sym).map.with_index.to_h
+      @c = json_load_ptr(obj["user_info_factors"])
+      @d = json_load_ptr(obj["item_info_factors"])
 
       # implicit features
-      @add_implicit_features = obj[:add_implicit_features]
-      @ai = load_ptr(obj[:user_factors_implicit])
-      @bi = load_ptr(obj[:item_factors_implicit])
+      @add_implicit_features = obj["add_implicit_features"]
+      @ai = json_load_ptr(obj["user_factors_implicit"])
+      @bi = json_load_ptr(obj["item_factors_implicit"])
 
       unless @implicit
-        @min_rating = obj[:min_rating]
-        @max_rating = obj[:max_rating]
+        @min_rating = obj["min_rating"]
+        @max_rating = obj["max_rating"]
       end
 
-      @u_colmeans = load_ptr(obj[:user_means])
+      @u_colmeans = json_load_ptr(obj["user_means"])
 
       @m = @user_map.size
       @n = @item_map.size

data/lib/cmfrec/version.rb

@@ -1,3 +1,3 @@
 module Cmfrec
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: cmfrec
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-15 00:00:00.000000000 Z
-dependencies: []
+date: 2024-10-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fiddle
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description:
 email: andrew@ankane.org
 executables: []
@@ -45,14 +59,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.5.16
 signing_key:
 specification_version: 4
 summary: Recommendations for Ruby using collective matrix factorization