llm_memory 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85f3330ed767cc28b5a3276a7772678c36726ce04562c184a4e6dcf287d14d1d
-  data.tar.gz: 3e1ae83b2517f7f80bb857de5024b62b886bb6a045428f2d40159e48818cf286
+  metadata.gz: 072b5fa983000a18bccb6e5d0185663cbb30547a0fd6cc6194e1d4235ae833fb
+  data.tar.gz: 7233ecf0cbbb254b08cbf7d1fe747b8374e1e3baa3b6214a1ba5fa64ecd0360d
 SHA512:
-  metadata.gz: dba158d07c4a97a5b2e6bb1a98987173ba427a18a1a06d0a422f366ab2f7335ac168aea0351657b0c7f957352e6ffe747a2d817deb7c8d5454010c2ec535999a
-  data.tar.gz: 1db3ccb501f3b56d6e6dc00997f3f085b74f395521ffbbc1775bc5799bc388410523f6f209cfc185efbbf1437177c07dd5ee1725a49f6a27b207c6fc296067aa
+  metadata.gz: 9e670ee60c3e780343317d2b2175ea5cf9fefdaa8b4ce81b13ec1b32caa3cbd13ad4363dec63ebaabf803e434897eb34a21d4b4e0b37b60bb4b3ff22b6a9a80c
+  data.tar.gz: e4defb25011469c10b36c6d4b98763dcb56a673eaedb2709741769de0fc888a8d30e4301f5444c88dbb07c31b8a290ae0fbfae584d9d93a21c6c7ce34d96c40b

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    llm_memory (0.1.3)
+    llm_memory (0.1.4)
       redis (~> 4.6.0)
       ruby-openai (~> 3.7.0)
       tiktoken_ruby (~> 0.0.4)

data/lib/llm_memory/hippocampus.rb

@@ -23,7 +23,7 @@ module LlmMemory
       raise "Store '#{store_name}' not found." unless store_class
       @store = store_class.new(index_name: index_name)
 
-      # word count, not char count
+      # char count, not word count
       @chunk_size = chunk_size
       @chunk_overlap = chunk_overlap
     end
@@ -87,18 +87,14 @@ module LlmMemory
       docs.each do |item|
         content = item[:content]
         metadata = item[:metadata]
-        words = content.split
-
-        if words.length > @chunk_size
+        if content.length > @chunk_size
           start_index = 0
-
-          while start_index < words.length
-            end_index = [start_index + @chunk_size, words.length].min
-            chunk_words = words[start_index...end_index]
-            chunk = chunk_words.join(" ")
+          while start_index < content.length
+            end_index = [start_index + @chunk_size, content.length].min
+            chunk = content[start_index...end_index]
             result << {content: chunk, metadata: metadata}
-
-            start_index += @chunk_size - @chunk_overlap # Move index to create a overlap
+            break if end_index == content.length
+            start_index += @chunk_size - @chunk_overlap
           end
         else
           result << {content: content, metadata: metadata}

data/lib/llm_memory/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LlmMemory
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llm_memory
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Shohei Kameda