classifier 1.4.4 → 2.1.0

data/lib/classifier/lsi.rb

@@ -1,20 +1,60 @@
+# rbs_inline: enabled
+
 # Author::    David Fayram  (mailto:dfayram@lensmen.net)
 # Copyright:: Copyright (c) 2005 David Fayram II
 # License::   LGPL
 
+module Classifier
+  class LSI
+    # Backend options: :native, :ruby
+    # @rbs @backend: Symbol
+    @backend = :ruby
+
+    class << self
+      # @rbs @backend: Symbol
+      attr_accessor :backend
+
+      # Check if using native C extension
+      # @rbs () -> bool
+      def native_available?
+        backend == :native
+      end
+
+      # Get the Vector class for the current backend
+      # @rbs () -> Class
+      def vector_class
+        backend == :native ? Classifier::Linalg::Vector : ::Vector
+      end
+
+      # Get the Matrix class for the current backend
+      # @rbs () -> Class
+      def matrix_class
+        backend == :native ? Classifier::Linalg::Matrix : ::Matrix
+      end
+    end
+  end
+end
+
+# Backend detection: native extension > pure Ruby
+# Set NATIVE_VECTOR=true to force pure Ruby implementation
+
 begin
-  # to test the native vector class, try `rake test NATIVE_VECTOR=true`
   raise LoadError if ENV['NATIVE_VECTOR'] == 'true'
 
-  require 'gsl' # requires https://github.com/SciRuby/rb-gsl/
-  require 'classifier/extensions/vector_serialize'
-  $GSL = true
+  require 'classifier/classifier_ext'
+  Classifier::LSI.backend = :native
 rescue LoadError
-  warn 'Notice: for 10x faster LSI support, please install https://github.com/SciRuby/rb-gsl/'
-  $GSL = false
+  # Fall back to pure Ruby implementation
+  unless ENV['SUPPRESS_LSI_WARNING'] == 'true'
+    warn 'Notice: for 5-10x faster LSI, install the classifier gem with native extensions. ' \
+         'Set SUPPRESS_LSI_WARNING=true to hide this.'
+  end
+  Classifier::LSI.backend = :ruby
   require 'classifier/extensions/vector'
 end
 
+require 'json'
+require 'mutex_m'
 require 'classifier/lsi/word_list'
 require 'classifier/lsi/content_node'
 require 'classifier/lsi/summary'
@@ -24,26 +64,62 @@ module Classifier
   # data based on underlying semantic relations. For more information on the algorithms used,
   # please consult Wikipedia[http://en.wikipedia.org/wiki/Latent_Semantic_Indexing].
   class LSI
-    attr_reader :word_list
-    attr_accessor :auto_rebuild
+    include Mutex_m
+
+    # @rbs @auto_rebuild: bool
+    # @rbs @word_list: WordList
+    # @rbs @items: Hash[untyped, ContentNode]
+    # @rbs @version: Integer
+    # @rbs @built_at_version: Integer
+    # @rbs @singular_values: Array[Float]?
+    # @rbs @dirty: bool
+    # @rbs @storage: Storage::Base?
+
+    attr_reader :word_list, :singular_values
+    attr_accessor :auto_rebuild, :storage
 
     # Create a fresh index.
     # If you want to call #build_index manually, use
-    #      Classifier::LSI.new :auto_rebuild => false
+    #      Classifier::LSI.new auto_rebuild: false
     #
+    # @rbs (?Hash[Symbol, untyped]) -> void
     def initialize(options = {})
+      super()
       @auto_rebuild = true unless options[:auto_rebuild] == false
       @word_list = WordList.new
       @items = {}
       @version = 0
       @built_at_version = -1
+      @dirty = false
+      @storage = nil
     end
 
     # Returns true if the index needs to be rebuilt.  The index needs
     # to be built after all informaton is added, but before you start
     # using it for search, classification and cluster detection.
+    #
+    # @rbs () -> bool
     def needs_rebuild?
-      (@items.keys.size > 1) && (@version != @built_at_version)
+      synchronize { (@items.keys.size > 1) && (@version != @built_at_version) }
+    end
+
+    # @rbs () -> Array[Hash[Symbol, untyped]]?
+    def singular_value_spectrum
+      return nil unless @singular_values
+
+      total = @singular_values.sum
+      return nil if total.zero?
+
+      cumulative = 0.0
+      @singular_values.map.with_index do |value, i|
+        cumulative += value
+        {
+          dimension: i,
+          value: value,
+          percentage: value / total,
+          cumulative_percentage: cumulative / total
+        }
+      end
     end
 
     # Adds an item to the index. item is assumed to be a string, but
@@ -59,10 +135,14 @@ module Classifier
     #   ar = ActiveRecordObject.find( :all )
     #   lsi.add_item ar, *ar.categories { |x| ar.content }
     #
+    # @rbs (String, *String | Symbol) ?{ (String) -> String } -> void
     def add_item(item, *categories, &block)
       clean_word_hash = block ? block.call(item).clean_word_hash : item.to_s.clean_word_hash
-      @items[item] = ContentNode.new(clean_word_hash, *categories)
-      @version += 1
+      synchronize do
+        @items[item] = ContentNode.new(clean_word_hash, *categories)
+        @version += 1
+        @dirty = true
+      end
       build_index if @auto_rebuild
     end
 
@@ -70,30 +150,42 @@ module Classifier
     # you are passing in a string with no categorries. item
     # will be duck typed via to_s .
     #
+    # @rbs (String) -> void
     def <<(item)
       add_item(item)
     end
 
     # Returns the categories for a given indexed items. You are free to add and remove
     # items from this as you see fit. It does not invalide an index to change its categories.
+    #
+    # @rbs (String) -> Array[String | Symbol]
     def categories_for(item)
-      return [] unless @items[item]
+      synchronize do
+        return [] unless @items[item]
 
-      @items[item].categories
+        @items[item].categories
+      end
     end
 
     # Removes an item from the database, if it is indexed.
     #
+    # @rbs (String) -> void
     def remove_item(item)
-      return unless @items.key?(item)
+      removed = synchronize do
+        next false unless @items.key?(item)
 
-      @items.delete(item)
-      @version += 1
+        @items.delete(item)
+        @version += 1
+        @dirty = true
+        true
+      end
+      build_index if removed && @auto_rebuild
     end
 
     # Returns an array of items that are indexed.
+    # @rbs () -> Array[untyped]
     def items
-      @items.keys
+      synchronize { @items.keys }
     end
 
     # This function rebuilds the index if needs_rebuild? returns true.
@@ -110,34 +202,33 @@ module Classifier
     # cutoff parameter tells the indexer how many of these values to keep.
     # A value of 1 for cutoff means that no semantic analysis will take place,
     # turning the LSI class into a simple vector search engine.
+    #
+    # @rbs (?Float) -> void
     def build_index(cutoff = 0.75)
-      return unless needs_rebuild?
-
-      make_word_list
-
-      doc_list = @items.values
-      tda = doc_list.collect { |node| node.raw_vector_with(@word_list) }
-
-      if $GSL
-        tdm = GSL::Matrix.alloc(*tda).trans
-        ntdm = build_reduced_matrix(tdm, cutoff)
-
-        ntdm.size[1].times do |col|
-          vec = GSL::Vector.alloc(ntdm.column(col)).row
-          doc_list[col].lsi_vector = vec
-          doc_list[col].lsi_norm = vec.normalize
+      validate_cutoff!(cutoff)
+
+      synchronize do
+        return unless needs_rebuild_unlocked?
+
+        make_word_list
+
+        doc_list = @items.values
+        tda = doc_list.collect { |node| node.raw_vector_with(@word_list) }
+
+        if self.class.native_available?
+          # Convert vectors to arrays for matrix construction
+          tda_arrays = tda.map { |v| v.respond_to?(:to_a) ? v.to_a : v }
+          tdm = self.class.matrix_class.alloc(*tda_arrays).trans
+          ntdm = build_reduced_matrix(tdm, cutoff)
+          assign_native_ext_lsi_vectors(ntdm, doc_list)
+        else
+          tdm = Matrix.rows(tda).trans
+          ntdm = build_reduced_matrix(tdm, cutoff)
+          assign_ruby_lsi_vectors(ntdm, doc_list)
         end
-      else
-        tdm = Matrix.rows(tda).trans
-        ntdm = build_reduced_matrix(tdm, cutoff)
 
-        ntdm.row_size.times do |col|
-          doc_list[col].lsi_vector = ntdm.column(col) if doc_list[col]
-          doc_list[col].lsi_norm = ntdm.column(col).normalize if doc_list[col]
-        end
+        @built_at_version = @version
       end
-
-      @built_at_version = @version
     end
 
     # This method returns max_chunks entries, ordered by their average semantic rating.
@@ -148,13 +239,17 @@ module Classifier
     # your dataset's general content. For example, if you were to use categorize on the
     # results of this data, you could gather information on what your dataset is generally
     # about.
+    #
+    # @rbs (?Integer) -> Array[String]
     def highest_relative_content(max_chunks = 10)
-      return [] if needs_rebuild?
+      synchronize do
+        return [] if needs_rebuild_unlocked?
 
-      avg_density = {}
-      @items.each_key { |x| avg_density[x] = proximity_array_for_content(x).inject(0.0) { |x, y| x + y[1] } }
+        avg_density = {}
+        @items.each_key { |x| avg_density[x] = proximity_array_for_content_unlocked(x).sum { |pair| pair[1] } }
 
-      avg_density.keys.sort_by { |x| avg_density[x] }.reverse[0..max_chunks - 1].map
+        avg_density.keys.sort_by { |x| avg_density[x] }.reverse[0..(max_chunks - 1)].map
+      end
     end
 
     # This function is the primitive that find_related and classify
@@ -169,20 +264,10 @@ module Classifier
     # The parameter doc is the content to compare. If that content is not
     # indexed, you can pass an optional block to define how to create the
     # text data. See add_item for examples of how this works.
+    #
+    # @rbs (String) ?{ (String) -> String } -> Array[[String, Float]]
     def proximity_array_for_content(doc, &block)
-      return [] if needs_rebuild?
-
-      content_node = node_for_content(doc, &block)
-      result =
-        @items.keys.collect do |item|
-          val = if $GSL
-                  content_node.search_vector * @items[item].search_vector.col
-                else
-                  (Matrix[content_node.search_vector] * @items[item].search_vector)[0]
-                end
-          [item, val]
-        end
-      result.sort_by { |x| x[1] }.reverse
+      synchronize { proximity_array_for_content_unlocked(doc, &block) }
     end
 
     # Similar to proximity_array_for_content, this function takes similar
@@ -190,20 +275,10 @@ module Classifier
     # calculated vectors instead of their full versions. This is useful when
     # you're trying to perform operations on content that is much smaller than
     # the text you're working with. search uses this primitive.
+    #
+    # @rbs (String) ?{ (String) -> String } -> Array[[String, Float]]
     def proximity_norms_for_content(doc, &block)
-      return [] if needs_rebuild?
-
-      content_node = node_for_content(doc, &block)
-      result =
-        @items.keys.collect do |item|
-          val = if $GSL
-                  content_node.search_norm * @items[item].search_norm.col
-                else
-                  (Matrix[content_node.search_norm] * @items[item].search_norm)[0]
-                end
-          [item, val]
-        end
-      result.sort_by { |x| x[1] }.reverse
+      synchronize { proximity_norms_for_content_unlocked(doc, &block) }
     end
 
     # This function allows for text-based search of your index. Unlike other functions
@@ -213,12 +288,16 @@ module Classifier
     #
     # While this may seem backwards compared to the other functions that LSI supports,
     # it is actually the same algorithm, just applied on a smaller document.
+    #
+    # @rbs (String, ?Integer) -> Array[String]
     def search(string, max_nearest = 3)
-      return [] if needs_rebuild?
+      synchronize do
+        return [] if needs_rebuild_unlocked?
 
-      carry = proximity_norms_for_content(string)
-      result = carry.collect { |x| x[0] }
-      result[0..max_nearest - 1]
+        carry = proximity_norms_for_content_unlocked(string)
+        result = carry.collect { |x| x[0] }
+        result[0..(max_nearest - 1)]
+      end
     end
 
     # This function takes content and finds other documents
@@ -230,11 +309,15 @@ module Classifier
     # This is particularly useful for identifing clusters in your document space.
     # For example you may want to identify several "What's Related" items for weblog
     # articles, or find paragraphs that relate to each other in an essay.
+    #
+    # @rbs (String, ?Integer) ?{ (String) -> String } -> Array[String]
     def find_related(doc, max_nearest = 3, &block)
-      carry =
-        proximity_array_for_content(doc, &block).reject { |pair| pair[0] == doc }
-      result = carry.collect { |x| x[0] }
-      result[0..max_nearest - 1]
+      synchronize do
+        carry =
+          proximity_array_for_content_unlocked(doc, &block).reject { |pair| pair[0] == doc }
+        result = carry.collect { |x| x[0] }
+        result[0..(max_nearest - 1)]
+      end
     end
 
     # This function uses a voting system to categorize documents, based on
@@ -242,30 +325,23 @@ module Classifier
     # find_related function to find related documents, then returns the
     # most obvious category from this list.
     #
-    # cutoff signifies the number of documents to consider when clasifying
-    # text. A cutoff of 1 means that every document in the index votes on
-    # what category the document is in. This may not always make sense.
-    #
+    # @rbs (String, ?Float) ?{ (String) -> String } -> String | Symbol
     def classify(doc, cutoff = 0.30, &block)
-      votes = vote(doc, cutoff, &block)
+      validate_cutoff!(cutoff)
+
+      synchronize do
+        votes = vote_unlocked(doc, cutoff, &block)
 
-      ranking = votes.keys.sort_by { |x| votes[x] }
-      ranking[-1]
+        ranking = votes.keys.sort_by { |x| votes[x] }
+        ranking[-1]
+      end
     end
 
+    # @rbs (String, ?Float) ?{ (String) -> String } -> Hash[String | Symbol, Float]
     def vote(doc, cutoff = 0.30, &block)
-      icutoff = (@items.size * cutoff).round
-      carry = proximity_array_for_content(doc, &block)
-      carry = carry[0..icutoff - 1]
-      votes = {}
-      carry.each do |pair|
-        categories = @items[pair[0]].categories
-        categories.each do |category|
-          votes[category] ||= 0.0
-          votes[category] += pair[1]
-        end
-      end
-      votes
+      validate_cutoff!(cutoff)
+
+      synchronize { vote_unlocked(doc, cutoff, &block) }
     end
 
     # Returns the same category as classify() but also returns
@@ -278,59 +354,337 @@ module Classifier
     #   category = nil
     # end
     #
-    #
     # See classify() for argument docs
+    # @rbs (String, ?Float) ?{ (String) -> String } -> [String | Symbol | nil, Float?]
     def classify_with_confidence(doc, cutoff = 0.30, &block)
-      votes = vote(doc, cutoff, &block)
-      votes_sum = votes.values.inject(0.0) { |sum, v| sum + v }
-      return [nil, nil] if votes_sum.zero?
+      validate_cutoff!(cutoff)
 
-      ranking = votes.keys.sort_by { |x| votes[x] }
-      winner = ranking[-1]
-      vote_share = votes[winner] / votes_sum.to_f
-      [winner, vote_share]
+      synchronize do
+        votes = vote_unlocked(doc, cutoff, &block)
+        votes_sum = votes.values.sum
+        return [nil, nil] if votes_sum.zero?
+
+        ranking = votes.keys.sort_by { |x| votes[x] }
+        winner = ranking[-1]
+        vote_share = votes[winner] / votes_sum.to_f
+        [winner, vote_share]
+      end
     end
 
     # Prototype, only works on indexed documents.
     # I have no clue if this is going to work, but in theory
     # it's supposed to.
+    # @rbs (String, ?Integer) -> Array[Symbol]
     def highest_ranked_stems(doc, count = 3)
-      raise 'Requested stem ranking on non-indexed content!' unless @items[doc]
+      synchronize do
+        raise 'Requested stem ranking on non-indexed content!' unless @items[doc]
+
+        arr = node_for_content_unlocked(doc).lsi_vector.to_a
+        top_n = arr.sort.reverse[0..(count - 1)]
+        top_n.collect { |x| @word_list.word_for_index(arr.index(x)) }
+      end
+    end
+
+    # Custom marshal serialization to exclude mutex state
+    # @rbs () -> Array[untyped]
+    def marshal_dump
+      [@auto_rebuild, @word_list, @items, @version, @built_at_version, @dirty]
+    end
+
+    # Custom marshal deserialization to recreate mutex
+    # @rbs (Array[untyped]) -> void
+    def marshal_load(data)
+      mu_initialize
+      @auto_rebuild, @word_list, @items, @version, @built_at_version, @dirty = data
+      @storage = nil
+    end
+
+    # Returns a hash representation of the LSI index.
+    # Only source data (word_hash, categories) is included, not computed vectors.
+    # This can be converted to JSON or used directly.
+    #
+    # @rbs () -> untyped
+    def as_json(*)
+      items_data = @items.transform_values do |node|
+        {
+          word_hash: node.word_hash.transform_keys(&:to_s),
+          categories: node.categories.map(&:to_s)
+        }
+      end
+
+      {
+        version: 1,
+        type: 'lsi',
+        auto_rebuild: @auto_rebuild,
+        items: items_data
+      }
+    end
+
+    # Serializes the LSI index to a JSON string.
+    # Only source data (word_hash, categories) is serialized, not computed vectors.
+    # On load, the index will be rebuilt automatically.
+    #
+    # @rbs () -> String
+    def to_json(*)
+      as_json.to_json
+    end
+
+    # Loads an LSI index from a JSON string or Hash created by #to_json or #as_json.
+    # The index will be rebuilt after loading.
+    #
+    # @rbs (String | Hash[String, untyped]) -> LSI
+    def self.from_json(json)
+      data = json.is_a?(String) ? JSON.parse(json) : json
+      raise ArgumentError, "Invalid classifier type: #{data['type']}" unless data['type'] == 'lsi'
+
+      # Create instance with auto_rebuild disabled during loading
+      instance = new(auto_rebuild: false)
+
+      # Restore items (categories stay as strings, matching original storage)
+      data['items'].each do |item_key, item_data|
+        word_hash = item_data['word_hash'].transform_keys(&:to_sym)
+        categories = item_data['categories']
+        instance.instance_variable_get(:@items)[item_key] = ContentNode.new(word_hash, *categories)
+        instance.instance_variable_set(:@version, instance.instance_variable_get(:@version) + 1)
+      end
 
-      arr = node_for_content(doc).lsi_vector.to_a
-      top_n = arr.sort.reverse[0..count - 1]
-      top_n.collect { |x| @word_list.word_for_index(arr.index(x)) }
+      # Restore auto_rebuild setting and rebuild index
+      instance.auto_rebuild = data['auto_rebuild']
+      instance.build_index
+      instance
+    end
+
+    # Saves the LSI index to the configured storage.
+    # Raises ArgumentError if no storage is configured.
+    #
+    # @rbs () -> void
+    def save
+      raise ArgumentError, 'No storage configured. Use save_to_file(path) or set storage=' unless storage
+
+      storage.write(to_json)
+      @dirty = false
+    end
+
+    # Saves the LSI index to a file (legacy API).
+    #
+    # @rbs (String) -> Integer
+    def save_to_file(path)
+      result = File.write(path, to_json)
+      @dirty = false
+      result
+    end
+
+    # Reloads the LSI index from the configured storage.
+    # Raises UnsavedChangesError if there are unsaved changes.
+    # Use reload! to force reload and discard changes.
+    #
+    # @rbs () -> self
+    def reload
+      raise ArgumentError, 'No storage configured' unless storage
+      raise UnsavedChangesError, 'Unsaved changes would be lost. Call save first or use reload!' if @dirty
+
+      data = storage.read
+      raise StorageError, 'No saved state found' unless data
+
+      restore_from_json(data)
+      @dirty = false
+      self
+    end
+
+    # Force reloads the LSI index from storage, discarding any unsaved changes.
+    #
+    # @rbs () -> self
+    def reload!
+      raise ArgumentError, 'No storage configured' unless storage
+
+      data = storage.read
+      raise StorageError, 'No saved state found' unless data
+
+      restore_from_json(data)
+      @dirty = false
+      self
+    end
+
+    # Returns true if there are unsaved changes.
+    #
+    # @rbs () -> bool
+    def dirty?
+      @dirty
+    end
+
+    # Loads an LSI index from the configured storage.
+    # The storage is set on the returned instance.
+    #
+    # @rbs (storage: Storage::Base) -> LSI
+    def self.load(storage:)
+      data = storage.read
+      raise StorageError, 'No saved state found' unless data
+
+      instance = from_json(data)
+      instance.storage = storage
+      instance
+    end
+
+    # Loads an LSI index from a file (legacy API).
+    #
+    # @rbs (String) -> LSI
+    def self.load_from_file(path)
+      from_json(File.read(path))
     end
 
     private
 
-    def build_reduced_matrix(matrix, cutoff = 0.75)
-      # TODO: Check that M>=N on these dimensions! Transpose helps assure this
-      u, v, s = matrix.SV_decomp
+    # Restores LSI state from a JSON string (used by reload)
+    # @rbs (String) -> void
+    def restore_from_json(json)
+      data = JSON.parse(json)
+      raise ArgumentError, "Invalid classifier type: #{data['type']}" unless data['type'] == 'lsi'
+
+      synchronize do
+        # Recreate the items
+        @items = {}
+        data['items'].each do |item_key, item_data|
+          word_hash = item_data['word_hash'].transform_keys(&:to_sym)
+          categories = item_data['categories']
+          @items[item_key] = ContentNode.new(word_hash, *categories)
+        end
 
-      # TODO: Better than 75% term, please. :\
-      s_cutoff = s.sort.reverse[(s.size * cutoff).round - 1]
-      s.size.times do |ord|
-        s[ord] = 0.0 if s[ord] < s_cutoff
+        # Restore settings
+        @auto_rebuild = data['auto_rebuild']
+        @version += 1
+        @built_at_version = -1
+        @word_list = WordList.new
+        @dirty = false
       end
-      # Reconstruct the term document matrix, only with reduced rank
-      u * ($GSL ? GSL::Matrix : ::Matrix).diag(s) * v.trans
+
+      # Rebuild the index
+      build_index
+    end
+
+    # @rbs (Float) -> void
+    def validate_cutoff!(cutoff)
+      return if cutoff.positive? && cutoff < 1
+
+      raise ArgumentError, "cutoff must be between 0 and 1 (exclusive), got #{cutoff}"
+    end
+
+    # Assigns LSI vectors using native C extension
+    # @rbs (untyped, Array[ContentNode]) -> void
+    def assign_native_ext_lsi_vectors(ntdm, doc_list)
+      ntdm.size[1].times do |col|
+        vec = self.class.vector_class.alloc(ntdm.column(col).to_a).row
+        doc_list[col].lsi_vector = vec
+        doc_list[col].lsi_norm = vec.normalize
+      end
+    end
+
+    # Assigns LSI vectors using pure Ruby Matrix
+    # @rbs (untyped, Array[ContentNode]) -> void
+    def assign_ruby_lsi_vectors(ntdm, doc_list)
+      ntdm.column_size.times do |col|
+        next unless doc_list[col]
+
+        column = ntdm.column(col)
+        next unless column
+
+        doc_list[col].lsi_vector = column
+        doc_list[col].lsi_norm = column.normalize
+      end
+    end
+
+    # Unlocked version of needs_rebuild? for internal use when lock is already held
+    # @rbs () -> bool
+    def needs_rebuild_unlocked?
+      (@items.keys.size > 1) && (@version != @built_at_version)
     end
 
-    def node_for_content(item, &block)
+    # Unlocked version of proximity_array_for_content for internal use
+    # @rbs (String) ?{ (String) -> String } -> Array[[String, Float]]
+    def proximity_array_for_content_unlocked(doc, &)
+      return [] if needs_rebuild_unlocked?
+
+      content_node = node_for_content_unlocked(doc, &)
+      result =
+        @items.keys.collect do |item|
+          val = if self.class.native_available?
+                  content_node.search_vector * @items[item].search_vector.col
+                else
+                  (Matrix[content_node.search_vector] * @items[item].search_vector)[0]
+                end
+          [item, val]
+        end
+      result.sort_by { |x| x[1] }.reverse
+    end
+
+    # Unlocked version of proximity_norms_for_content for internal use
+    # @rbs (String) ?{ (String) -> String } -> Array[[String, Float]]
+    def proximity_norms_for_content_unlocked(doc, &)
+      return [] if needs_rebuild_unlocked?
+
+      content_node = node_for_content_unlocked(doc, &)
+      result =
+        @items.keys.collect do |item|
+          val = if self.class.native_available?
+                  content_node.search_norm * @items[item].search_norm.col
+                else
+                  (Matrix[content_node.search_norm] * @items[item].search_norm)[0]
+                end
+          [item, val]
+        end
+      result.sort_by { |x| x[1] }.reverse
+    end
+
+    # Unlocked version of vote for internal use
+    # @rbs (String, ?Float) ?{ (String) -> String } -> Hash[String | Symbol, Float]
+    def vote_unlocked(doc, cutoff = 0.30, &)
+      icutoff = (@items.size * cutoff).round
+      carry = proximity_array_for_content_unlocked(doc, &)
+      carry = carry[0..(icutoff - 1)]
+      votes = {}
+      carry.each do |pair|
+        categories = @items[pair[0]].categories
+        categories.each do |category|
+          votes[category] ||= 0.0
+          votes[category] += pair[1]
+        end
+      end
+      votes
+    end
+
+    # Unlocked version of node_for_content for internal use
+    # @rbs (String) ?{ (String) -> String } -> ContentNode
+    def node_for_content_unlocked(item, &block)
       return @items[item] if @items[item]
 
       clean_word_hash = block ? block.call(item).clean_word_hash : item.to_s.clean_word_hash
+      cn = ContentNode.new(clean_word_hash, &block)
+      cn.raw_vector_with(@word_list) unless needs_rebuild_unlocked?
+      cn
+    end
+
+    # @rbs (untyped, ?Float) -> untyped
+    def build_reduced_matrix(matrix, cutoff = 0.75)
+      # TODO: Check that M>=N on these dimensions! Transpose helps assure this
+      u, v, s = matrix.SV_decomp
 
-      cn = ContentNode.new(clean_word_hash, &block) # make the node and extract the data
+      @singular_values = s.sort.reverse
 
-      unless needs_rebuild?
-        cn.raw_vector_with(@word_list) # make the lsi raw and norm vectors
+      s_cutoff_index = [(s.size * cutoff).round - 1, 0].max
+      s_cutoff = @singular_values[s_cutoff_index]
+      s.size.times do |ord|
+        s[ord] = 0.0 if s[ord] < s_cutoff
       end
+      # Reconstruct the term document matrix, only with reduced rank
+      result = u * self.class.matrix_class.diag(s) * v.trans
 
-      cn
+      # SVD may return transposed dimensions when row_size < column_size
+      # Ensure result matches input dimensions
+      result = result.trans if result.row_size != matrix.row_size
+
+      result
     end
 
+    # @rbs () -> void
     def make_word_list
       @word_list = WordList.new
       @items.each_value do |node|