valkey-objects 0.3.7 → 0.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6796a90c43779a0ba50c73cdd3d4f3240cd2c311b6e26aad64791d82b69cbbe5
-  data.tar.gz: d10c65e147d7964afb31163deda067f5d27e67ba1427c150286f34ca29ea2ae4
+  metadata.gz: 773f8c7d573ecf3a521ccf195d14d669cf627badaf03499fd936670cafe33356
+  data.tar.gz: 9aed29f7bf5bf476f6d6a8d18a3dae5c16828656336df706edd80d08ac4a0cd3
 SHA512:
-  metadata.gz: d4ddc3a86f521f6591501e0aea1fd84161332a237a2a5500dfdcdd94df0138efef94fe0581313941eb58030050615b3fd3c7a4c104c3447a715dc6550d462858
-  data.tar.gz: 9f0593942ebd888802b9484c12ce95ee7edc0119b3e37b24b4df4797f8cf180f60a921e3871cc48f0fea268ad9addfe3d068f4457d9897715e7e4cac49b79b12
+  metadata.gz: 55c3959e92a716d40bc6813254940023afadf25f6a64bfb26e92f5a73197cf5c75572a7c6eeac08e0e062b5b3a7aab6b9071c818fb35b41f26b9dd8c4ebdd3d3
+  data.tar.gz: 382540a624ce59c47149acfe0455dac4ba6ed8328c7808eb6394f072add9a2d95afbf6bc09069bc15401c674a0c5c4c94b26c996339486a7d15e3754b20f87e3

data/lib/valkey/objects/version.rb

@@ -2,6 +2,6 @@
 
 module Valkey
   module Objects
-    VERSION = "0.3.7"
+    VERSION = "0.3.9"
   end
 end

data/lib/valkey/objects.rb

@@ -11,6 +11,7 @@ require 'yaml'
 require 'classifier-reborn'
 require 'knn'
 require 'sentimental'
+require 'tokenizer'
 
 module VK
 
@@ -181,7 +182,23 @@ module VK
     # @obj.myVector.value { |i,e| ... }
     # @obj.myvector[0] = "An Entry of Text."
     define_method(:vector) { |k, h={}| define_method(k.to_sym) { VECTOR.new(%[#{xx}:vector:#{k}:#{@id}], h); } };
-    
+    ##
+    # An ordered list of strings.
+    ##
+    # vector :myCorpus
+    # @obj.myCorpus
+    # @obj.myCorpus.value { |i,e| ... }
+    # @obj.myCorpus["An entry"] => 5
+    define_method(:corpus) { |k, h={}| define_method(k.to_sym) { CORPUS.new(%[#{xx}:corpus:#{k}:#{@id}], h); } }; 
+    ##
+    # An ordered list of strings.
+    ##
+    # vector :myBook
+    # @obj.myBook
+    # @obj.myBook.value { |index,entry,embedding| ... }
+    # @obj.mybook.near("Some entry") => [...]
+    # @obj.myBook["An entry"] => 5
+    define_method(:book) { |k, h={}| define_method(k.to_sym) { BOOK.new(%[#{xx}:book:#{k}:#{@id}], h); } };    
   end
   
   def id
@@ -614,6 +631,73 @@ module VK
     end
   end
 
+  class CORPUS < O
+    def _set
+		%[#{key}-sset]
+    end
+    def _index
+		%[#{key}-hash]
+    end
+	def length
+	  VK.redis.call("SCARD", _set)
+	end
+	def value &b
+		VK.redis.call("HGETALL", _index).each_pair { |e, i| b.call(i, e) }
+	end
+	def [] i			
+		if VK.redis.call("SMISMEMBER", _set, i)[0] == 0
+			VK.redis.call("SADD", _set, i)
+			VK.redis.call("HSET", _index, i, length);
+		end
+		return VK.redis.call("HGET", _index, i)
+	end
+  end
+
+  class BOOK < O
+    def _set
+		%[#{key}-sset]
+    end
+    def _index
+		%[#{key}-index]
+    end
+    def _sec
+		%[#{key}-sec]
+    end    
+    def _embed
+		%[#{key}-embed]
+    end
+	def length
+	  VK.redis.call("SCARD", _set)
+	end
+	def value &b
+		VK.redis.call("HGETALL", _index).each_pair { |e, i| b.call(i.to_i, e) }
+	end
+	def entry
+		Hash.new { |h,k| VK.redis.call("HGET", _sec, k) }
+	end
+	def vectors
+		a = []
+		value { |i, e| a << [ i, VK.embed(e) ].flatten }		
+		return a
+	end
+	def embed
+		return KNN.new(vectors, :distance_measure => :tanimoto_coefficient)
+	end
+	def near g, *n
+		 return embed.nearest_neighbours(VK.embed(g)).map { |e|
+			{ index: e[0], distance: e[1], entry: entry[e[0]] }
+		 }.sort_by { |ee| ee[:distance] }
+	end
+	def [] i			
+		if VK.redis.call("SMISMEMBER", _set, i.to_s)[0] == 0
+			VK.redis.call("HSET", _index, i.to_s, length.to_s);
+			VK.redis.call("HSET", _sec, length.to_s, i.to_s);
+			VK.redis.call("SADD", _set, i)
+		end
+		return VK.redis.call("HGET", _index, i.to_s).to_i;
+	end
+  end
+
 	@@SENTIMENT_THRESHOLD = 0.9
 	##
 	# Bayesean Classification
@@ -663,19 +747,30 @@ module VK
 	# index[:vector]['Text to classify'] => :index
 	# index[:vector].learn("Text to classify", :index)
 	# index[:vector].search("Similar text.", 5) => ["Similar text?", "other similar text", ...]
-	def self.cluster
+	def self.index
 		@@CRi
 	end
-	##
-	# Vector Engine
-	#
-	# Return known string based upon nearest neighbor by vector of number.
-	##
-	# vector([["return val", 0, 1, 2, ...], ...]).classify(["guess", 0, 1, 2, ...]) => "return val"
-	def self.cluster vectors, size
-		Knn::Classifier.new(vectors, SquaredEuclideanCalculator)
+	@@VI = Hash.new { |h,k| h[k] = [] }
+	@@VC = Hash.new { |h,k| h[k] = Knn::Classifier.new(@@VI[k], SquaredEuclideanCalculator) }
+	def self.learn fact, index
+		@@CRi[index.to_sym].learn fact, index
+		@@VI[index] << fact
 	end
 
+	@@TOKENIZER = Tokenizer::WhitespaceTokenizer.new(:en)
+  def self.tokenize i
+  	@@TOKENIZER.tokenize(i)	
+  end
+
+  @@WORDLIST = CORPUS.new(%[WORDLIST])
+  def self.wordlist
+	@@WORDLIST
+  end
+
+  def self.embed i
+	m = VK.tokenize(i).map { |e| VK.wordlist[e].to_f }
+	return [m, Array.new(2048 - m.length, 0.0)].flatten
+  end
   
   def self.flushdb!
     VK.redis.call("FLUSHDB")
@@ -684,4 +779,13 @@ module VK
   def self.[] k
     VK.redis.call("KEYS", k)
   end
+
+  def self.txt f, &b
+	x = File.read(f).gsub(/\n\n+/,"\n\n").gsub(/  +/, " ").split("\n\n").map { |e| e.gsub(/\n/, " ") }
+	if block_given?
+		x.map { |e| b.call(e) }
+	else
+		return x
+	end
+  end
 end

data/valkey-objects.gemspec

@@ -38,8 +38,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "ruby-duration"
   spec.add_dependency "classifier-reborn"
   spec.add_dependency "amatch"
-  spec.add_dependency "knn"
+  spec.add_dependency "naive_bayes"
   spec.add_dependency "sentimental"
+  spec.add_dependency "tokenizer"
   
   spec.add_dependency "pry"
   spec.add_dependency "awesome_print"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: valkey-objects
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.3.9
 platform: ruby
 authors:
 - Erik Olson
@@ -94,7 +94,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: knn
+  name: naive_bayes
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -121,6 +121,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: tokenizer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement