llm_memory 0.1.14 → 0.1.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1ddc205359564a6863d027d91243d6da9cc2079a8965a89c3e0dde0fe13c32cb
-  data.tar.gz: 23a4980aa97684abcf9893c5598446243ec5b9c198c4c083a11d7cd25884c34c
+  metadata.gz: 54df42766547d29c56e8396eaa9e83182eaee3ec48798ad9ad68a981ba3d6243
+  data.tar.gz: 8082ab64110019e9d2313046029737ed5447004f999e8ad4a90891af960ffbc7
 SHA512:
-  metadata.gz: 03df070d8832990b091c5c4814632e23c832ba595eceaaaa769a6cd06823d302783bd5b1865b6b5531cd13d07287abeb9ed7a31296d73efd6a3031738502074b
-  data.tar.gz: 9b46d3197283516e44aec1de34799a0ae50b9232f00b6c55f462b8a529c8de9e38d2b6971b0eb9d09056a03cb4f6978787783921deca018863b8437accf37dc2
+  metadata.gz: 500f5f1c58ea2507b8b40bb5aa0b6b3c168559f070d9abfb59e183befeb48c2b5e57de6e85a6b7af7e126296734ca91db042108c2c2b25ab2722ea71809e632f
+  data.tar.gz: f33e32af701b590a507cde6571c2717d0b762025b164ff396032830f2d1cd855d221977dec9eb539021ee2c6e6305598d2423799b21bf74ab3ca6852f8b189b3

data/.ruby-version

@@ -0,0 +1 @@
+3.2.3

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    llm_memory (0.1.14)
+    llm_memory (0.1.15)
       redis (~> 4.6.0)
       ruby-openai (~> 3.7.0)
-      tokenizers (~> 0.3.3)
+      tokenizers (~> 0.6.1)
 
 GEM
   remote: https://rubygems.org/
@@ -87,8 +87,8 @@ GEM
     standard-performance (1.0.1)
       lint_roller (~> 1.0)
       rubocop-performance (~> 1.16.0)
-    tokenizers (0.3.3-arm64-darwin)
-    tokenizers (0.3.3-x86_64-linux)
+    tokenizers (0.6.1-arm64-darwin)
+    tokenizers (0.6.1-x86_64-linux)
     unicode-display_width (2.4.2)
     vcr (6.1.0)
     webmock (3.18.1)
@@ -98,6 +98,7 @@ GEM
 
 PLATFORMS
   arm64-darwin-22
+  arm64-darwin-23
   x86_64-linux
 
 DEPENDENCIES
@@ -115,4 +116,4 @@ DEPENDENCIES
   webmock (~> 3.18.1)
 
 BUNDLED WITH
-   2.4.6
+   2.4.19

data/docs/how_to_use_ragas.md

@@ -0,0 +1,118 @@
+At this point of wriging this readme (Nov 20, 2023), we use RAGAS to evaluate RAG. To use ragas from llm_memory, we do the following steps.
+
+#### 1. Prepare test data with the following format
+
+```
+test_data = [
+ {
+    question: "Is it better to stay with my child during lessons, or should they be left alone?",
+    contexts: [],
+    answer: "",
+    ground_truths: ["To improve English proficiency, we recommend that children take lessons by themselves as much as possible. However, if your child is unable to concentrate, we ask that parents support their child. If you need to facilitate smooth progress in the lesson (for instance, by informing us of your child's interests to help with teacher-student conversations), please consult with the Student Relations department."]
+ },
+ # Prepare multiple question and groud_truths. 10 ~ 20 (Note that the groud truth is not mandatory)
+]
+```
+
+#### 2. Use llm_memory to fill the `contexts` and `answer`
+
+```
+test_data = [...] 
+
+hippocampus = LlmMemory::Hippocampus.new(index_name: "sr_emails")  
+
+prompt = <<-TEMPLATE
+    # YOUR PROMT
+    <%= query_str %>
+
+    Use the following info to respond
+    ---------------------
+    <% related_docs.each do |doc| %>
+    <%= doc[:content] %>
+    <% end %>
+    ---------------------
+    TEMPLATE
+
+test_data.each{|td| 
+  related_docs = hippocampus.query(td[:question], limit: 10)
+  td[:contexts] = related_docs
+ 
+    broca = LlmMemory::Broca.new(
+      prompt: prompt, 
+      model: "gpt-4",
+      temperature: 0,
+      max_token: 8192
+    )
+    message = broca.respond(query_str: td[:question], related_docs: related_docs)
+    td[:answer] = message
+}
+```
+
+#### 3. Dump to file like JSON or YAML 
+
+```
+File.open('out.yaml', 'w') do |f|  f.write(test_data.to_yaml) end
+system('curl -F "file=@out.yaml" https://file.io') # heroku to upload the file
+```
+
+#### 4. Install python/ragas
+
+Set python environment if it's not there yet.
+
+```
+$ pip install ragas
+```
+
+#### 5. Create a python script to do the evaluation
+
+```
+import yaml
+from datasets import Dataset
+from ragas import evaluate
+import os
+
+# Load the YAML file
+file_path = 'out.yaml'
+with open(file_path, 'r') as file:
+    yaml_data = yaml.safe_load(file)
+
+# Updated parsing function to create a dictionary with lists for each column
+def parse_data(yaml_data):
+    # Initialize a dictionary with keys and empty lists
+    parsed_data = {
+        'question': [],
+        'contexts': [],
+        'answer': [],
+        'ground_truths': []
+    }
+
+    # Populate the dictionary
+    for item in yaml_data:
+        parsed_data['question'].append(item[':question'])
+        parsed_data['contexts'].append([context[':content'] for context in item[':contexts']])
+        parsed_data['answer'].append(item[':answer'])
+        parsed_data['ground_truths'].append(item[':ground_truths'])
+
+    return parsed_data
+
+# Convert the YAML data to the required format
+parsed_yaml_data = parse_data(yaml_data)
+
+# Create the dataset
+dataset = Dataset.from_dict(parsed_yaml_data)
+for row in dataset:
+    print(row)
+
+# Run RAGAS evaluation
+results = evaluate(dataset)
+print(results)
+```
+
+#### 5. Execute it
+Name the previous script as `evaluate.py` and run it with the environment variable.
+
+```
+$ RAGAS_DO_NOT_TRACK=true OPENAI_API_KEY=YOUR_KEY python evaluate.py
+...
+{'answer_relevancy': 0.7575, 'context_precision': 0.5574, 'faithfulness': 0.7167, 'context_recall': 0.1250}  
+```

data/lib/llm_memory/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LlmMemory
-  VERSION = "0.1.14"
+  VERSION = "0.1.15"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: llm_memory
 version: !ruby/object:Gem::Version
-  version: 0.1.14
+  version: 0.1.15
 platform: ruby
 authors:
 - Shohei Kameda
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-01 00:00:00.000000000 Z
+date: 2025-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tokenizers
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 0.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 0.6.1
 - !ruby/object:Gem::Dependency
   name: ruby-openai
   requirement: !ruby/object:Gem::Requirement
@@ -63,6 +63,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".rspec"
+- ".ruby-version"
 - ".standard.yml"
 - ".vscode/settings.json"
 - CHANGELOG.md
@@ -72,6 +73,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- docs/how_to_use_ragas.md
 - lib/llm_memory.rb
 - lib/llm_memory/broca.rb
 - lib/llm_memory/configuration.rb
@@ -85,7 +87,6 @@ files:
 - lib/llm_memory/stores/redis_store.rb
 - lib/llm_memory/version.rb
 - lib/llm_memory/wernicke.rb
-- llm_memory.gemspec
 - sig/llm_memory.rbs
 homepage: https://github.com/shohey1226/llm_memory
 licenses:
@@ -109,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.6
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: A Ruby Gem for LLMs like ChatGPT to have memory using in-context learning

data/llm_memory.gemspec

@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "lib/llm_memory/version"
-
-Gem::Specification.new do |spec|
-  spec.name = "llm_memory"
-  spec.version = LlmMemory::VERSION
-  spec.authors = ["Shohei Kameda"]
-  spec.email = ["shoheik@cpan.org"]
-
-  spec.summary = "A Ruby Gem for LLMs like ChatGPT to have memory using in-context learning"
-  spec.description = "LLM Memory is a Ruby gem designed to provide large language models (LLMs) like ChatGPT with memory using in-context learning. This enables better integration with systems such as Rails and web services while providing a more user-friendly and abstract interface based on brain terms."
-  spec.homepage = "https://github.com/shohey1226/llm_memory"
-  spec.license = "MIT"
-  spec.required_ruby_version = ">= 2.6.0"
-
-  spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "https://github.com/shohey1226/llm_memory"
-  spec.metadata["changelog_uri"] = "https://github.com/shohey1226/llm_memory/CHANGELOG.md"
-
-  # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files = Dir.chdir(__dir__) do
-    `git ls-files -z`.split("\x0").reject do |f|
-      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|circleci)|appveyor)})
-    end
-  end
-  spec.bindir = "exe"
-  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-
-  # Uncomment to register a new dependency of your gem
-  # spec.add_dependency "example-gem", "~> 1.0"
-  spec.add_dependency "tokenizers", "~> 0.3.3"
-  spec.add_dependency "ruby-openai", "~> 3.7.0"
-  spec.add_dependency "redis", "~> 4.6.0"
-
-  # For more information and examples about making a new gem, check out our
-  # guide at: https://bundler.io/guides/creating_gem.html
-end