disco 0.2.8 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ec370f448c5cc8aebb2860b0580466687874eb34c165e1a3d0254a1c6e701d7
-  data.tar.gz: 1b080c37206371ee59ce184ae420c5fa1de60da0714ef17ea9459b31fcdd22ab
+  metadata.gz: a4af4d7df56f884618557fd98f97da2686cecbfaf3ce1f1f52b6ba1a3a9155f5
+  data.tar.gz: ddbc7551c3534c41284e958042a94d988cdd5c52248f3ca7e4a8d8a72c6b168e
 SHA512:
-  metadata.gz: 1802e0fbf68ee489891f94e468c3b24df0eb8463de4e55a7f50a9dbc86bda7631ca15d5b84d0d584dbd341e994db84a82f00f6753c70eda4a7400376f0443df5
-  data.tar.gz: bee0645357a5fc4eb226e75d4b56208e7377952911dd19ccad8f3072eee3eaf4d4a318933dcb4ec4d75273630e278dbf98b3ac21f5987baea90261d15cc2d851
+  metadata.gz: 5f40f125fe4096dcf09eaf1d1295f23e68fef9bbe1e5651a1ecfa1ed06748df7a0f4c9ea048b767b429b67b4d0add1f32663388707616571a6936a55ffd1d6b7
+  data.tar.gz: baf3caa4deec5422bd85e9372bc717f683d4cbb067f468eb7d4c964556885f7bb273fa0b6339201aef917c8f046346c20356358462fc73103243608a5245ebe6

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## 0.3.1 (2022-07-10)
+
+- Added support for JSON serialization
+
+## 0.3.0 (2022-03-22)
+
+- Changed `item_id` to `user_id` for `similar_users`
+- Changed warning to an error when `value` passed to `fit`
+- Changed to use Faiss over NGT for `optimize_item_recs` and `optimize_similar_users` when both are installed
+- Removed dependency on `wilson_score` gem for `top_items`
+- Dropped support for Ruby < 2.6
+
+## 0.2.9 (2022-03-22)
+
+- Fixed error with `load_movielens`
+
 ## 0.2.8 (2022-03-13)
 
 - Fixed error with `top_items` with all same rating

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2019-2021 Andrew Kane
+Copyright (c) 2019-2022 Andrew Kane
 
 MIT License
 

data/README.md

@@ -13,7 +13,7 @@
 Add this line to your application’s Gemfile:
 
 ```ruby
-gem 'disco'
+gem "disco"
 ```
 
 ## Getting Started
@@ -183,17 +183,19 @@ For Rails < 6, speed up inserts by adding [activerecord-import](https://github.c
 If you’d prefer to perform recommendations on-the-fly, store the recommender
 
 ```ruby
-bin = Marshal.dump(recommender)
-File.binwrite("recommender.bin", bin)
+json = recommender.to_json
+File.write("recommender.json", json)
 ```
 
-> You can save it to a file, database, or any other storage system
+> You can save it to a file, database, or any other storage system. Also, user and item IDs should be integers or strings for this.
+
+The serialized recommender includes user activity from the training data (to avoid recommending previously rated items), so be sure to protect it.
 
 Load a recommender
 
 ```ruby
-bin = File.binread("recommender.bin")
-recommender = Marshal.load(bin)
+json = File.read("recommender.json")
+recommender = Disco::Recommender.load_json(json)
 ```
 
 Alternatively, you can store only the factors and use a library like [Neighbor](https://github.com/ankane/neighbor). See the [examples](https://github.com/ankane/neighbor/tree/master/examples).
@@ -242,7 +244,7 @@ recommender.fit(data)
 recommender.top_items
 ```
 
-This uses [Wilson score](https://www.evanmiller.org/how-not-to-sort-by-average-rating.html) for explicit feedback (add [wilson_score](https://github.com/instacart/wilson_score) to your application’s Gemfile) and item frequency for implicit feedback.
+This uses [Wilson score](https://www.evanmiller.org/how-not-to-sort-by-average-rating.html) for explicit feedback and item frequency for implicit feedback.
 
 ## Data
 
@@ -271,7 +273,7 @@ If you have a large number of users or items, you can use an approximate nearest
 Add this line to your application’s Gemfile:
 
 ```ruby
-gem 'faiss'
+gem "faiss"
 ```
 
 Speed up the `user_recs` method with:

data/lib/disco/data.rb

@@ -1,9 +1,11 @@
 module Disco
   module Data
     def load_movielens
-      item_path = download_file("ml-100k/u.item", "http://files.grouplens.org/datasets/movielens/ml-100k/u.item",
+      require "csv"
+
+      item_path = download_file("ml-100k/u.item", "https://files.grouplens.org/datasets/movielens/ml-100k/u.item",
         file_hash: "553841ebc7de3a0fd0d6b62a204ea30c1e651aacfb2814c7a6584ac52f2c5701")
-      data_path = download_file("ml-100k/u.data", "http://files.grouplens.org/datasets/movielens/ml-100k/u.data",
+      data_path = download_file("ml-100k/u.data", "https://files.grouplens.org/datasets/movielens/ml-100k/u.data",
         file_hash: "06416e597f82b7342361e41163890c81036900f418ad91315590814211dca490")
 
       # convert u.item to utf-8
@@ -29,6 +31,11 @@ module Disco
     private
 
     def download_file(fname, origin, file_hash:)
+      require "digest"
+      require "fileutils"
+      require "net/http"
+      require "tmpdir"
+
       # TODO handle this better
       raise "No HOME" unless ENV["HOME"]
       dest = "#{ENV["HOME"]}/.disco/#{fname}"

data/lib/disco/model.rb

@@ -10,10 +10,9 @@ module Disco
 
         has_many :"recommended_#{name}", -> { where("disco_recommendations.context = ?", name).order("disco_recommendations.score DESC") }, through: :recommendations, source: :item, source_type: class_name
 
-        # TODO use fetch for item_id and score in 0.3.0
         define_method("update_recommended_#{name}") do |items|
           now = Time.now
-          items = items.map { |item| {subject_type: model_name.name, subject_id: id, item_type: class_name, item_id: item[:item_id], context: name, score: item[:score], created_at: now, updated_at: now} }
+          items = items.map { |item| {subject_type: model_name.name, subject_id: id, item_type: class_name, item_id: item.fetch(:item_id), context: name, score: item.fetch(:score), created_at: now, updated_at: now} }
 
           self.class.transaction do
             recommendations.where(context: name).delete_all

data/lib/disco/recommender.rb

@@ -23,7 +23,7 @@ module Disco
       @implicit = !train_set.any? { |v| v[:rating] }
 
       if @implicit && train_set.any? { |v| v[:value] }
-        warn "[disco] WARNING: Passing `:value` with implicit feedback has no effect on recommendations and can be removed. Earlier versions of the library incorrectly stated this was used."
+        raise ArgumentError, "Passing `:value` with implicit feedback has no effect on recommendations and should be removed. Earlier versions of the library incorrectly stated this was used."
       end
 
       # TODO improve performance
@@ -167,13 +167,13 @@ module Disco
 
     def similar_items(item_id, count: 5)
       check_fit
-      similar(item_id, @item_map, normalized_item_factors, count, @similar_items_index)
+      similar(item_id, :item_id, @item_map, normalized_item_factors, count, @similar_items_index)
     end
     alias_method :item_recs, :similar_items
 
     def similar_users(user_id, count: 5)
       check_fit
-      similar(user_id, @user_map, normalized_user_factors, count, @similar_users_index)
+      similar(user_id, :user_id, @user_map, normalized_user_factors, count, @similar_users_index)
     end
 
     def top_items(count: 5)
@@ -183,27 +183,20 @@ module Disco
       if @implicit
         scores = Numo::UInt64.cast(@item_count)
       else
-        require "wilson_score"
+        min_rating = @min_rating
 
-        range =
-          if @min_rating == @max_rating
-            # TODO remove temp fix
-            (@min_rating - 1)..@max_rating
-          else
-            @min_rating..@max_rating
-          end
-        scores = Numo::DFloat.cast(@item_sum.zip(@item_count).map { |s, c| WilsonScore.rating_lower_bound(s / c, c, range) })
+        # TODO remove temp fix
+        min_rating -= 1 if @min_rating == @max_rating
 
-        # TODO uncomment in 0.3.0
         # wilson score with continuity correction
         # https://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval#Wilson_score_interval_with_continuity_correction
-        # z = 1.96 # 95% confidence
-        # range = @max_rating - @min_rating
-        # n = Numo::DFloat.cast(@item_count)
-        # phat = (Numo::DFloat.cast(@item_sum) - (@min_rating * n)) / range / n
-        # phat = (phat - (1 / (2 * n))).clip(0, nil) # continuity correction
-        # scores = (phat + z**2 / (2 * n) - z * Numo::DFloat::Math.sqrt((phat * (1 - phat) + z**2 / (4 * n)) / n)) / (1 + z**2 / n)
-        # scores = scores * range + @min_rating
+        z = 1.96 # 95% confidence
+        range = @max_rating - @min_rating
+        n = Numo::DFloat.cast(@item_count)
+        phat = (Numo::DFloat.cast(@item_sum) - (min_rating * n)) / range / n
+        phat = (phat - (1 / (2 * n))).clip(0, nil) # continuity correction
+        scores = (phat + z**2 / (2 * n) - z * Numo::DFloat::Math.sqrt((phat * (1 - phat) + z**2 / (4 * n)) / n)) / (1 + z**2 / n)
+        scores = scores * range + min_rating
       end
 
       indexes = scores.sort_index.reverse
@@ -262,12 +255,51 @@ module Disco
       to_s # for now
     end
 
+    def to_json
+      require "base64"
+      require "json"
+
+      obj = {
+        implicit: @implicit,
+        user_ids: @user_map.keys,
+        item_ids: @item_map.keys,
+        rated: @user_map.map { |_, u| (@rated[u] || {}).keys },
+        global_mean: @global_mean,
+        user_factors: Base64.strict_encode64(@user_factors.to_binary),
+        item_factors: Base64.strict_encode64(@item_factors.to_binary),
+        factors: @factors,
+        epochs: @epochs,
+        verbose: @verbose
+      }
+
+      unless @implicit
+        obj[:min_rating] = @min_rating
+        obj[:max_rating] = @max_rating
+      end
+
+      if @top_items
+        obj[:item_count] = @item_count
+        obj[:item_sum] = @item_sum
+      end
+
+      JSON.generate(obj)
+    end
+
+    def self.load_json(json)
+      require "json"
+
+      obj = JSON.parse(json)
+
+      recommender = new
+      recommender.send(:json_load, obj)
+      recommender
+    end
+
     private
 
     # factors should already be normalized for similar users/items
     def create_index(factors, library:)
-      # TODO make Faiss the default in 0.3.0
-      library ||= defined?(Faiss) && !defined?(Ngt) ? "faiss" : "ngt"
+      library ||= defined?(Ngt) && !defined?(Faiss) ? "ngt" : "faiss"
 
       case library
       when "faiss"
@@ -276,7 +308,7 @@ module Disco
         # inner product is cosine similarity with normalized vectors
         # https://github.com/facebookresearch/faiss/issues/95
         #
-        # TODO use non-exact index in 0.3.0
+        # TODO add option for index type
         # https://github.com/facebookresearch/faiss/wiki/Faiss-indexes
         # index = Faiss::IndexHNSWFlat.new(factors.shape[1], 32, :inner_product)
         index = Faiss::IndexFlatIP.new(factors.shape[1])
@@ -318,7 +350,7 @@ module Disco
       factors / norms.expand_dims(1)
     end
 
-    def similar(id, map, norm_factors, count, index)
+    def similar(id, key, map, norm_factors, count, index)
       i = map[id]
 
       if i && norm_factors.shape[0] > 1
@@ -342,9 +374,6 @@ module Disco
 
         keys = map.keys
 
-        # TODO use user_id for similar_users in 0.3.0
-        key = :item_id
-
         result = []
         # items can have the same score
         # so original item may not be at index 0
@@ -445,5 +474,31 @@ module Disco
         @item_sum = obj[:item_sum]
       end
     end
+
+    def json_load(obj)
+      require "base64"
+
+      @implicit = obj["implicit"]
+      @user_map = obj["user_ids"].map.with_index.to_h
+      @item_map = obj["item_ids"].map.with_index.to_h
+      @rated = obj["rated"].map.with_index.to_h { |r, i| [i, r.to_h { |v| [v, true] }] }
+      @global_mean = obj["global_mean"].to_f
+      @factors = obj["factors"].to_i
+      @user_factors = Numo::SFloat.from_binary(Base64.strict_decode64(obj["user_factors"]), [@user_map.size, @factors])
+      @item_factors = Numo::SFloat.from_binary(Base64.strict_decode64(obj["item_factors"]), [@item_map.size, @factors])
+      @epochs = obj["epochs"].to_i
+      @verbose = obj["verbose"]
+
+      unless @implicit
+        @min_rating = obj["min_rating"]
+        @max_rating = obj["max_rating"]
+      end
+
+      @top_items = obj.key?("item_count")
+      if @top_items
+        @item_count = obj["item_count"]
+        @item_sum = obj["item_sum"]
+      end
+    end
   end
 end

data/lib/disco/version.rb

@@ -1,3 +1,3 @@
 module Disco
-  VERSION = "0.2.8"
+  VERSION = "0.3.1"
 end

data/lib/disco.rb

@@ -2,11 +2,6 @@
 require "libmf"
 require "numo/narray"
 
-# stdlib
-require "csv"
-require "fileutils"
-require "net/http"
-
 # modules
 require "disco/data"
 require "disco/metrics"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: disco
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.3.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-13 00:00:00.000000000 Z
+date: 2022-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: libmf
@@ -69,7 +69,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="