rag_embeddings 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cd1dc0ac570ef83c9c79142cb516411454137919c649ab7f446e18748f7f7717
+  data.tar.gz: d21f7e9b2eee1324b4a0f7a5d892d3f55f1767bb42a965e0828225ba823461b8
+SHA512:
+  metadata.gz: 3fd776d3ff4b3082eac778534f2fc56aa541ea048d70d69ddbfb2738da9fb252012d6d7ffa4af0f2a1fbed229ac86960ada844cd062a4c8294ecd4999b4cd0ed
+  data.tar.gz: b2682f1d217d9689a73e78fdbcfd74973de598f323dc12a35868530cb95e0833a584d29468d91c2e6e01afcf66585fe18f8388e54cee04b71b8e7dc955a10181

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Marco Mastrodonato
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,119 @@
+# 💎 Rag Embeddings  
+
+[![Gem Version](https://badge.fury.io/rb/rag_embeddings.svg)](https://badge.fury.io/rb/rag_embeddings)
+
+**rag_embeddings** is a native Ruby library for efficient storage and comparison of AI-generated embedding vectors (float arrays) using high-performance C extensions. It is designed for seamless integration with external LLMs (Ollama, OpenAI, Mistral, etc) and works perfectly for RAG (Retrieval-Augmented Generation) applications.
+
+- **C extension for maximum speed** in cosine similarity and vector allocation
+- **Compatible with langchainrb** for embedding generation (Ollama, OpenAI, etc)
+- **SQLite-based storage** with vector search capabilities
+- **RSpec tested**
+
+---
+
+## 📦 Features
+
+- Creation of embedding objects from LLM-generated float arrays
+- Cosine similarity calculation in C for speed and safety
+- Embedding + text storage in SQLite (BLOB)
+- Retrieve top-K most similar texts to a query using cosine similarity
+- Memory-safe and 100% Ruby compatible
+- Plug-and-play for RAG, semantic search, and retrieval AI
+
+---
+
+## 🔧 Installation
+
+Add to your Gemfile:
+
+```ruby
+gem "rag_embeddings"
+gem "langchainrb"
+gem "faraday"
+gem "sqlite3"
+```
+
+bundle install
+rake compile
+
+(Requires a working C compiler!)
+
+## 🏁 Running the test suite
+
+To run all specs (RSpec required):
+
+`bundle exec rspec`
+
+## 🧪 Practical examples
+
+### 1. Generate an embedding from text
+
+```ruby
+require "rag_embeddings"
+
+text = "Hello world, this is RAG!"
+embedding = RagEmbeddings.embed(text)
+# embedding is a float array
+```
+
+### 2. Create a C embedding object
+
+```ruby
+c_embedding = RagEmbeddings::Embedding.from_array(embedding)
+puts "Dimension: #{c_embedding.dim}"
+puts "Ruby array: #{c_embedding.to_a.inspect}"
+```
+
+### 3. Compute similarity between two texts
+
+```ruby
+emb1 = RagEmbeddings.embed("Hello world!")
+emb2 = RagEmbeddings.embed("Hi universe!")
+obj1 = RagEmbeddings::Embedding.from_array(emb1)
+obj2 = RagEmbeddings::Embedding.from_array(emb2)
+sim = obj1.cosine_similarity(obj2)
+puts "Cosine similarity: #{sim}"  # Value between -1 and 1
+```
+
+### 4. Store and search embeddings in a database
+
+```ruby
+db = RagEmbeddings::Database.new("embeddings.db")
+db.insert("Hello world!", RagEmbeddings.embed("Hello world!"))
+db.insert("Completely different sentence", RagEmbeddings.embed("Completely different sentence"))
+
+# Find the most similar text to a query
+result = db.top_k_similar("Hello!", k: 1)
+puts "Most similar text: #{result.first[1]}, score: #{result.first[2]}"
+```
+
+## 🏗️ How it works
+
+- Embeddings are managed as dynamic C objects for efficiency (variable dimension).
+- The only correct way to construct an embedding object is using .from_array.
+- Langchainrb integration lets you easily change the embedding provider (Ollama, OpenAI, etc).
+- Storage uses local SQLite with embeddings as BLOB, for maximum portability and simplicity.
+
+## 🎛️ Customization
+
+- Embedding provider: switch model/provider in engine.rb (Ollama, OpenAI, etc)
+- Database: set the SQLite file path as desired
+
+## 👷 Requirements
+
+- Ruby >= 3.3
+- langchainrb (for embedding)
+- sqlite3 (for storage)
+- A working C compiler
+
+## 📑 Notes
+
+- Always create embeddings with .from_array
+- All memory management is idiomatic and safe
+- For millions of vectors, consider vector DBs (Faiss, sqlite-vss, etc.)
+
+## 📬 Contact & Issues
+Open an issue or contact the maintainer for questions, suggestions, or bugs.
+
+
+Happy RAG! 🚀

data/Rakefile

@@ -0,0 +1,6 @@
+task :compile do
+  Dir.chdir("ext/rag_embeddings") do
+    ruby "extconf.rb"
+    system("make")
+  end
+end

data/ext/rag_embeddings/embedding.c

@@ -0,0 +1,117 @@
+#include <ruby.h>     // Ruby API
+#include <stdint.h>   // For integer types like uint16_t
+#include <stdlib.h>   // For memory allocation functions
+#include <math.h>     // For math functions like sqrt
+
+// Main data structure for storing embeddings
+// Flexible array member (values[]) allows variable length arrays
+typedef struct {
+  uint16_t dim;       // Dimension of the embedding vector
+  float values[];     // Flexible array member to store the actual values
+} embedding_t;
+
+// Callback for freeing memory when Ruby's GC collects our object
+static void embedding_free(void *ptr) {
+  xfree(ptr);         // Ruby's memory free function
+}
+
+// Callback to report memory usage to Ruby's GC
+static size_t embedding_memsize(const void *ptr) {
+  const embedding_t *emb = (const embedding_t *)ptr;
+  return emb ? sizeof(embedding_t) + emb->dim * sizeof(float) : 0;
+}
+
+// Type information for Ruby's GC:
+// Tells Ruby how to manage our C data structure
+static const rb_data_type_t embedding_type = {
+  "RagEmbeddings/Embedding",               // Type name
+  {0, embedding_free, embedding_memsize,}, // Functions: mark, free, size
+  0, 0,                                    // Parent type, data
+  RUBY_TYPED_FREE_IMMEDIATELY              // Flags
+};
+
+// Class method: RagEmbeddings::Embedding.from_array([1.0, 2.0, ...])
+// Creates a new embedding from a Ruby array
+static VALUE embedding_from_array(VALUE klass, VALUE rb_array) {
+  Check_Type(rb_array, T_ARRAY);           // Ensure argument is a Ruby array
+  uint16_t dim = (uint16_t)RARRAY_LEN(rb_array);
+
+  // Allocate memory for struct + array of floats
+  embedding_t *ptr = xmalloc(sizeof(embedding_t) + dim * sizeof(float));
+  ptr->dim = dim;
+
+  // Copy values from Ruby array to our C array
+  for (int i = 0; i < dim; ++i)
+    ptr->values[i] = (float)NUM2DBL(rb_ary_entry(rb_array, i));
+
+  // Wrap our C struct in a Ruby object
+  VALUE obj = TypedData_Wrap_Struct(klass, &embedding_type, ptr);
+  return obj;
+}
+
+// Instance method: embedding.dim
+// Returns the dimension of the embedding
+static VALUE embedding_dim(VALUE self) {
+  embedding_t *ptr;
+  // Get the C struct from the Ruby object
+  TypedData_Get_Struct(self, embedding_t, &embedding_type, ptr);
+  return INT2NUM(ptr->dim);
+}
+
+// Instance method: embedding.to_a
+// Converts the embedding back to a Ruby array
+static VALUE embedding_to_a(VALUE self) {
+  embedding_t *ptr;
+  TypedData_Get_Struct(self, embedding_t, &embedding_type, ptr);
+
+  // Create a new Ruby array with pre-allocated capacity
+  VALUE arr = rb_ary_new2(ptr->dim);
+
+  // Copy each float value to the Ruby array
+  for (int i = 0; i < ptr->dim; ++i)
+    rb_ary_push(arr, DBL2NUM(ptr->values[i]));
+
+  return arr;
+}
+
+// Instance method: embedding.cosine_similarity(other_embedding)
+// Calculate cosine similarity between two embeddings
+static VALUE embedding_cosine_similarity(VALUE self, VALUE other) {
+  embedding_t *a, *b;
+  // Get C structs for both embeddings
+  TypedData_Get_Struct(self, embedding_t, &embedding_type, a);
+  TypedData_Get_Struct(other, embedding_t, &embedding_type, b);
+
+  // Ensure dimensions match
+  if (a->dim != b->dim)
+    rb_raise(rb_eArgError, "Dimension mismatch");
+
+  float dot = 0.0f, norm_a = 0.0f, norm_b = 0.0f;
+
+  // Calculate dot product and vector magnitudes
+  for (int i = 0; i < a->dim; ++i) {
+    dot += a->values[i] * b->values[i];      // Dot product
+    norm_a += a->values[i] * a->values[i];   // Square of magnitude for vector a
+    norm_b += b->values[i] * b->values[i];   // Square of magnitude for vector b
+  }
+
+  // Apply cosine similarity formula: dot(a,b)/(|a|*|b|)
+  // Small epsilon (1e-8) added to prevent division by zero
+  return DBL2NUM(dot / (sqrt(norm_a) * sqrt(norm_b) + 1e-8));
+}
+
+// Ruby extension initialization function
+// This function is called when the extension is loaded
+void Init_embedding(void) {
+  // Define module and class
+  VALUE mRag = rb_define_module("RagEmbeddings");
+  VALUE cEmbedding = rb_define_class_under(mRag, "Embedding", rb_cObject);
+
+  // Register class methods
+  rb_define_singleton_method(cEmbedding, "from_array", embedding_from_array, 1);
+
+  // Register instance methods
+  rb_define_method(cEmbedding, "dim", embedding_dim, 0);
+  rb_define_method(cEmbedding, "to_a", embedding_to_a, 0);
+  rb_define_method(cEmbedding, "cosine_similarity", embedding_cosine_similarity, 1);
+}

data/ext/rag_embeddings/extconf.rb

@@ -0,0 +1,2 @@
+require "mkmf"
+create_makefile("rag_embeddings/embedding")

data/lib/rag_embeddings/database.rb

@@ -0,0 +1,39 @@
+require "sqlite3"
+
+module RagEmbeddings
+  class Database
+    def initialize(path = "embeddings.db")
+      @db = SQLite3::Database.new(path)
+      @db.execute <<~SQL
+        CREATE TABLE IF NOT EXISTS embeddings (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          content TEXT NOT NULL,
+          embedding BLOB NOT NULL
+        );
+      SQL
+    end
+
+    def insert(text, embedding)
+      blob = embedding.pack("f*")
+      @db.execute("INSERT INTO embeddings (content, embedding) VALUES (?, ?)", [text, blob])
+    end
+
+    def all
+      @db.execute("SELECT id, content, embedding FROM embeddings").map do |id, content, blob|
+        [id, content, blob.unpack("f*")]
+      end
+    end
+
+    # "Raw" search: returns the N texts most similar to the query
+    def top_k_similar(query_text, k: 5)
+      query_embedding = RagEmbeddings.embed(query_text)
+      query_obj = RagEmbeddings::Embedding.from_array(query_embedding)
+
+      all.map do |id, content, emb|
+        emb_obj = RagEmbeddings::Embedding.from_array(emb)
+        similarity = emb_obj.cosine_similarity(query_obj)
+        [id, content, similarity]
+      end.sort_by { |_,_,sim| -sim }.first(k)
+    end
+  end
+end

data/lib/rag_embeddings/engine.rb

@@ -0,0 +1,13 @@
+require "langchainrb"
+
+module RagEmbeddings
+  MODEL = "gemma3".freeze
+
+  def self.llm
+    @llm ||= Langchain::LLM::Ollama.new(url: "http://localhost:11434", default_options: { temperature: 0.1, model: MODEL })
+  end
+
+  def self.embed(text)
+    llm.embed(text: text).embedding
+  end
+end

data/lib/rag_embeddings/version.rb

@@ -0,0 +1,3 @@
+module RagEmbeddings
+  VERSION = "0.1.0"
+end

data/lib/rag_embeddings.rb

@@ -0,0 +1,4 @@
+require_relative "rag_embeddings/version"
+require_relative "rag_embeddings/engine"
+require_relative "rag_embeddings/database"
+require_relative "../ext/rag_embeddings/embedding" # Loads the compiled C extension

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: rag_embeddings
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Marco Mastrodonato
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: langchainrb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Manage AI vector embeddings in C with Ruby integration
+email:
+- m.mastrodonato@gmail.com
+executables: []
+extensions:
+- ext/rag_embeddings/extconf.rb
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- Rakefile
+- ext/rag_embeddings/embedding.c
+- ext/rag_embeddings/extconf.rb
+- lib/rag_embeddings.rb
+- lib/rag_embeddings/database.rb
+- lib/rag_embeddings/engine.rb
+- lib/rag_embeddings/version.rb
+homepage: https://rubygems.org/gems/rag_embeddings
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://rubygems.org/gems/rag_embeddings
+  source_code_uri: https://github.com/marcomd/rag_embeddings
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Efficient RAG embedding storage and retrieval
+test_files: []