robot_lab-document_store 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fe6aeaf2a0fd6a0c1dd5b827a9382e3804cf9cf6e8573fdd9c75063fccd87a36
+  data.tar.gz: 350a44adfb40048c467166a1bfc37dd169941a35c4f5e3dd56ace185e39ba27b
+SHA512:
+  metadata.gz: 05bc51a7b278d57d7bf8b1bb8535d5520ac41093112b0080c0f2f6e7eb554101d212b8c3065564f25a031ce3439f7b8ad9562fa93a9ef9a4091d78f3d20bdea2
+  data.tar.gz: 62694a390bbcd9a2492466eef6102fd821ae670ca30b694283d25f73f4fa7c504e9c62e2e32509ebc2f3f7700d63499e74153111bd781bf190ea39c76f714137

data/.envrc

@@ -0,0 +1 @@
+export RR=`pwd`

data/.github/workflows/deploy-github-pages.yml

@@ -0,0 +1,52 @@
+name: Deploy Documentation to GitHub Pages
+on:
+  push:
+    branches:
+      - main
+      - develop
+    paths:
+      - "docs/**"
+      - "mkdocs.yml"
+      - ".github/workflows/deploy-github-pages.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pages: write
+  id-token: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+
+      - name: Install dependencies
+        run: |
+          pip install mkdocs
+          pip install mkdocs-material
+          pip install mkdocs-macros-plugin
+          pip install mike
+
+      - name: Configure Git
+        run: |
+          git config --local user.email "action@github.com"
+          git config --local user.name "GitHub Action"
+
+      - name: Build MkDocs site
+        run: mkdocs build
+
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./site
+          keep_files: true

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2026-05-07
+
+- Initial release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2026 Dewayne VanHoozer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,90 @@
+# robot_lab-document_store
+
+Embedding-based semantic document search for the [RobotLab](https://github.com/MadBomber/robot_lab) LLM agent framework.
+
+> [!CAUTION]
+> This gem is under active development. APIs may change without notice.
+
+## What it provides
+
+`RobotLab::DocumentStore` is a thread-safe, in-memory vector store backed by [fastembed](https://github.com/Anush008/fastembed-ruby) embeddings and cosine similarity search. It supports:
+
+- **`store(key, text)`** — embed and store a document under a symbol key
+- **`search(query, limit:)`** — return the top-N most similar documents by cosine similarity
+- **`delete(key)`** / **`clear`** — remove individual entries or wipe the store
+- **Asymmetric embedding** — passage embeddings for storage, query embeddings for retrieval
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem "robot_lab-document_store"
+```
+
+## Quick Example
+
+```ruby
+require "robot_lab/document_store"
+
+store = RobotLab::DocumentStore.new
+
+store.store(:alpha, "Ruby is a dynamic, open source programming language.")
+store.store(:beta,  "Python is widely used in data science and machine learning.")
+store.store(:gamma, "JavaScript runs in the browser and on Node.js servers.")
+
+results = store.search("What language is popular for AI?", limit: 2)
+results.each do |r|
+  puts "#{r[:key]} (score: #{"%.3f" % r[:score]})"
+end
+# => beta (score: 0.872)
+# => alpha (score: 0.641)
+```
+
+## Custom Model
+
+```ruby
+store = RobotLab::DocumentStore.new(
+  model_name: "BAAI/bge-small-en-v1.5"
+)
+```
+
+The default model is `"BAAI/bge-base-en-v1.5"`.
+
+## Using with RobotLab Robots
+
+`DocumentStore` works well as in-memory retrieval for RAG (retrieval-augmented generation) workflows. Load documents at startup and pass relevant excerpts into robot context:
+
+```ruby
+require "robot_lab"
+require "robot_lab/document_store"
+
+store = RobotLab::DocumentStore.new
+store.store(:faq_1, "Our return policy allows returns within 30 days.")
+store.store(:faq_2, "Shipping typically takes 3-5 business days.")
+
+robot = RobotLab.build(
+  name: "support",
+  system_prompt: "You are a support agent. Use provided context to answer questions."
+)
+
+query  = "How long do I have to return an item?"
+chunks = store.search(query, limit: 2).map { |r| r[:text] }.join("\n")
+
+result = robot.run("Context:\n#{chunks}\n\nQuestion: #{query}")
+puts result.last_text_content
+```
+
+## Links
+
+- [RobotLab Core](https://github.com/MadBomber/robot_lab)
+- [fastembed-ruby](https://github.com/Anush008/fastembed-ruby)
+- [RubyGems](https://rubygems.org/gems/robot_lab-document_store)
+
+## License
+
+MIT License - Copyright (c) 2025 Dewayne VanHoozer
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/MadBomber/robot_lab-document_store.

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "minitest/test_task"
+
+Minitest::TestTask.create
+
+task default: :test

data/docs/index.md

@@ -0,0 +1,58 @@
+# robot_lab-document_store
+
+Embedding-based semantic document search for the [RobotLab](https://github.com/MadBomber/robot_lab) LLM agent framework.
+
+> [!CAUTION]
+> This gem is under active development. APIs may change without notice.
+
+## What it provides
+
+`RobotLab::DocumentStore` is a thread-safe, in-memory vector store backed by [fastembed](https://github.com/Anush008/fastembed-ruby) embeddings and cosine similarity search. It supports:
+
+- **`store(key, text)`** — embed and store a document under a symbol key
+- **`search(query, limit:)`** — return the top-N most similar documents by cosine similarity
+- **`delete(key)`** / **`clear`** — remove individual entries or wipe the store
+- **Asymmetric embedding** — passage embeddings for storage, query embeddings for retrieval
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem "robot_lab-document_store"
+```
+
+## Quick Example
+
+```ruby
+require "robot_lab/document_store"
+
+store = RobotLab::DocumentStore.new
+
+store.store(:alpha, "Ruby is a dynamic, open source programming language.")
+store.store(:beta,  "Python is widely used in data science and machine learning.")
+store.store(:gamma, "JavaScript runs in the browser and on Node.js servers.")
+
+results = store.search("What language is popular for AI?", limit: 2)
+results.each do |r|
+  puts "#{r[:key]} (score: #{"%.3f" % r[:score]})"
+end
+# => beta (score: 0.872)
+# => alpha (score: 0.641)
+```
+
+## Custom Model
+
+```ruby
+store = RobotLab::DocumentStore.new(
+  model_name: "BAAI/bge-small-en-v1.5"
+)
+```
+
+The default model is `"BAAI/bge-base-en-v1.5"`.
+
+## Links
+
+- [RobotLab Core](https://github.com/MadBomber/robot_lab)
+- [fastembed-ruby](https://github.com/Anush008/fastembed-ruby)
+- [RubyGems](https://rubygems.org/gems/robot_lab-document_store)

data/examples/26_document_store/api_versioning_adr.md

@@ -0,0 +1,52 @@
+# Architecture Decision Record #047 — API Versioning Strategy
+
+**Status:** Accepted (2024-11-12)
+**Deciders:** Platform team, Mobile team, Partner integrations team
+
+## Context
+
+The v1 API has accumulated 23 breaking changes held back by an informal freeze
+while three external partners built integrations. The mobile apps ship on a
+4-week release cycle and cannot deploy hotfixes to force users to upgrade. We
+need a versioning strategy that allows the backend to evolve without coordinated
+lockstep releases across all consumers.
+
+## Decision
+
+We adopt URI-based versioning (/api/v2/, /api/v3/) rather than header-based
+(Accept: application/vnd.company.v2+json) for the following reasons:
+
+- URI versioning is visible in logs, dashboards, and browser dev tools.
+- Proxy and CDN rules can target specific version prefixes.
+- Internal clients are all first-party and can be updated in lockstep.
+
+Header-based versioning is reserved for minor non-breaking variants (e.g.,
+adding optional fields) using the Prefer header.
+
+## Support Lifecycle
+
+Each major version is supported for 18 months from GA. Deprecation notices are
+added to response headers (Sunset: date) 6 months before EOL. The deprecation
+dashboard tracks call volume per version per consumer; we do not retire a
+version with > 100 calls/day without direct partner outreach.
+
+## Backwards Compatibility Rules
+
+Within a version, we **may**:
+- Add new fields to responses.
+- Add new optional request parameters.
+- Add new endpoints.
+- Add new enum values (consumers must ignore unknown values).
+
+We **must not**:
+- Remove or rename fields.
+- Change field types.
+- Change HTTP status codes for existing success cases.
+- Remove endpoints.
+
+## Migration Tooling
+
+A version compatibility shim layer translates v1 requests to v2 internal
+representations and back-translates responses. This allows v1 to remain
+operational without duplicating business logic. The shim is tested with a
+contract test suite against recorded v1 response fixtures.

data/examples/26_document_store/incident_postmortem.md

@@ -0,0 +1,46 @@
+# Incident Postmortem — INC-2024-089
+
+**Date:** 2024-10-03
+**Duration:** 47 minutes
+**Severity:** P1
+**Affected:** API gateway, order processing, checkout flows
+
+## Timeline
+
+| Time  | Event |
+|-------|-------|
+| 14:23 | Automated alert fires: p99 API latency exceeds 5 seconds |
+| 14:25 | On-call engineer pages in; confirms checkout error rate at 34% |
+| 14:31 | Identified spike in slow queries on orders table in Datadog APM |
+| 14:38 | Root cause confirmed: migration added non-concurrent index at peak traffic |
+| 14:44 | DBA kills the migration process; index creation aborted |
+| 14:48 | Query latency returns to baseline; error rate drops to 0.2% |
+| 15:10 | Full recovery confirmed; incident closed |
+
+## Root Cause
+
+An engineer ran a schema migration that created an index on orders.status
+without the CONCURRENTLY keyword. Postgres acquired an AccessExclusiveLock on
+the orders table for the duration of the index build (11 minutes). All queries
+touching the orders table queued behind the lock, exhausting the PgBouncer
+connection pool within 3 minutes.
+
+## Contributing Factors
+
+1. Migration review checklist did not include "concurrent index" verification.
+2. The migration was run manually during business hours, not via the deploy pipeline.
+3. No automated linting (strong_migrations) was enforced in CI.
+
+## Remediation (Completed)
+
+- `strong_migrations` gem added to Gemfile; CI fails on unsafe migration patterns.
+- Runbook updated: all migrations that touch tables > 1M rows require DBA review.
+- Index creation added to the concurrent-operations checklist.
+- PgBouncer max_client_conn increased from 150 to 300 as a buffer.
+
+## Lessons Learned
+
+Lock acquisition during index creation is silent in application logs — the first
+visible symptom is connection pool exhaustion, not a database error.
+Instrumenting pg_locks with an alert on long-held AccessExclusiveLocks would
+have cut detection time from 8 minutes to under 1 minute.

data/examples/26_document_store/postgres_runbook.md

@@ -0,0 +1,49 @@
+# PostgreSQL Operations Runbook — v3.1
+
+## Slow Query Investigation
+
+When a query exceeds 1 second, start with pg_stat_statements:
+
+    SELECT query, mean_exec_time, calls, total_exec_time
+    FROM pg_stat_statements ORDER BY mean_exec_time DESC LIMIT 20;
+
+Use EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT) on the top offenders.
+Look for Sequential Scans on large tables (> 50k rows) and Hash Joins on
+unindexed foreign keys. Missing index candidates appear as "rows removed by
+filter" values that are an order of magnitude larger than the rows returned.
+
+## Connection Pool Exhaustion
+
+PgBouncer pools connections at the transaction level. When all connections are
+in use, new queries queue until pool_size is reached, at which point clients
+receive "too many clients" errors. Mitigate by:
+1. Reducing max_connections per Rails process via database.yml pool setting.
+2. Increasing server_pool_size in pgbouncer.ini incrementally.
+3. Identifying and killing idle-in-transaction connections:
+
+       SELECT pid, state, query, now() - query_start AS duration
+       FROM pg_stat_activity WHERE state = 'idle in transaction'
+       AND query_start < now() - interval '30 seconds';
+
+## Table Bloat and Vacuum
+
+High update/delete workloads generate table bloat. Check with:
+
+    SELECT relname, n_dead_tup, n_live_tup,
+           round(n_dead_tup::numeric / nullif(n_live_tup, 0) * 100, 1) AS dead_pct
+    FROM pg_stat_user_tables ORDER BY dead_pct DESC;
+
+If dead_pct exceeds 20% on a hot table, trigger VACUUM ANALYZE manually. For
+severe bloat, schedule an off-hours VACUUM FULL (acquires exclusive lock).
+Autovacuum scale factor defaults to 0.2; reduce to 0.05 on high-churn tables.
+
+## Replication Lag
+
+Monitor standby lag with:
+
+    SELECT client_addr, write_lag, flush_lag, replay_lag
+    FROM pg_stat_replication;
+
+Lag above 30 seconds indicates the replica is falling behind writes. Common
+causes: long-running VACUUM on primary holding WAL files, network saturation
+between primary and replica, or index builds on the replica.

data/examples/26_document_store/redis_caching_guide.md

@@ -0,0 +1,48 @@
+# Redis Caching Patterns — Implementation Guide
+
+## Cache Key Design
+
+Keys must encode every dimension that affects the cached value. For a
+user-scoped collection: `orders:user_USER_ID:page_PAGE:v2`. Always include a
+version suffix (v2) so a code deploy can invalidate globally by bumping the
+version, without a manual cache flush. Avoid encoding mutable data (e.g.,
+user.plan) directly in the key; use separate keys and join at read time,
+or accept stale reads.
+
+## TTL Strategy
+
+Set TTLs based on acceptable staleness, not on intuition:
+
+- User session data: 24h (refreshed on activity)
+- API response cache (authenticated): 5 minutes
+- API response cache (public, CDN-backed): 60 seconds
+- Computed aggregates (dashboards): 15 minutes with background refresh
+- Feature flags: 30 seconds (fast propagation of flag changes)
+
+Always set a TTL. Unbounded keys are a production outage waiting to happen
+when a runaway process fills the Redis instance.
+
+## Cache Invalidation
+
+Explicit invalidation is more reliable than TTL-only for write-heavy data. Use
+after_commit callbacks to delete or update cache entries when records change.
+For collections, track the latest updated_at timestamp as the cache key
+component (Russian doll caching). When multiple cache entries must be
+invalidated atomically, use a Redis pipeline or Lua script.
+
+## Redis Memory Pressure
+
+When Redis hits maxmemory, it evicts keys according to the eviction policy. Use
+`allkeys-lru` for pure cache workloads. Monitor `evicted_keys` in Redis INFO; a
+non-zero and growing value means your cache is too small for the working set.
+Separate cache and session data into different Redis instances (or databases)
+so session eviction cannot be triggered by cache pressure.
+
+## Stampede Protection
+
+Under high read concurrency, a cache miss causes multiple processes to
+simultaneously recompute the same expensive value — the cache stampede.
+Mitigate with probabilistic early expiration: recompute when TTL drops below a
+random fraction of the original TTL. Alternatively, use a distributed lock
+(Redlock or a simple SET NX PX lock key) to allow only one process to recompute
+while others wait briefly on the stale value.

data/examples/26_document_store/sidekiq_guide.md

@@ -0,0 +1,51 @@
+# Background Job Processing with Sidekiq — Engineering Guide
+
+## Job Design Principles
+
+Every Sidekiq job must be idempotent: running it twice with the same arguments
+must produce the same outcome. This is non-negotiable because Sidekiq retries
+failed jobs and at-least-once delivery is guaranteed, not exactly-once. Achieve
+idempotency by checking preconditions (has this invoice already been generated?),
+using database unique constraints on job output records, and passing Stripe
+idempotency keys.
+
+## Retry Configuration
+
+The default retry count is 25, which provides backoff up to ~21 days. For
+time-sensitive jobs (send_welcome_email) reduce to 3. For financial jobs
+(charge_subscription) raise to 15 to survive multi-hour outages.
+
+Configure per-job: `sidekiq_options retry: 10`
+
+Customize backoff with sidekiq_retry_in:
+
+    sidekiq_retry_in { |count| (count ** 4) + 15 + rand(30) * count }
+
+This gives approximately: 15s, 1m, 5m, 17m, 34m for the first 5 retries.
+
+## Circuit Breaker Pattern
+
+When a downstream service (Stripe, SendGrid) is degraded, jobs fail rapidly and
+fill the retry queue, creating a thundering-herd effect when the service
+recovers. Use a circuit breaker backed by Redis:
+
+- Set `stripe:circuit_open` in Redis when 3 consecutive failures occur.
+- In a job middleware, check the flag; if open, re-enqueue with 5-minute delay.
+- Auto-clear the flag after 10 minutes using Redis TTL.
+
+This converts retry churn into scheduled bursts.
+
+## Dead Queue Management
+
+Jobs reach the dead queue after exhausting all retries. Never bulk-retry
+blindly. Group dead jobs by error class, inspect a sample for root cause,
+fix the underlying issue, then use a Rake task to re-enqueue in batches of 50
+with a 1-second inter-batch sleep to avoid overwhelming the recovered service.
+Log each re-enqueue with original args and failure reason.
+
+## Queue Priority and Latency Budgets
+
+Define at least three queues: critical (< 1s SLA: auth, payments), default
+(< 30s: email, webhooks), and bulk (< 1h: exports, reports). Run dedicated
+Sidekiq processes per queue tier. Never mix critical and bulk work in the same
+process — a spike of bulk jobs will starve critical work if they share a queue.

data/examples/26_document_store.rb

@@ -0,0 +1,146 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Example 26: Embedding-Based Document Store
+#
+# Demonstrates Memory#store_document and Memory#search_documents — a
+# lightweight RAG store backed by fastembed (BAAI/bge-small-en-v1.5).
+#
+# Documents are multi-paragraph engineering guides stored as Markdown files in:
+#   examples/26_document_store/
+#
+# Usage:
+#   ruby examples/26_document_store.rb
+#   (Downloads the ~23 MB ONNX model on first run; cached afterwards.)
+
+require "robot_lab"
+require "robot_lab/document_store"
+
+puts "=" * 60
+puts "Example 26: Embedding-Based Document Store"
+puts "=" * 60
+puts
+puts "Note: First run downloads the fastembed model (~23 MB, cached)."
+puts
+
+# ---------------------------------------------------------------------------
+# Load documents from the companion directory
+# ---------------------------------------------------------------------------
+DOC_DIR = File.join(__dir__, "26_document_store")
+
+DOCUMENTS = Dir[File.join(DOC_DIR, "*.md")].sort.each_with_object({}) do |path, h|
+  key = File.basename(path, ".md").to_sym
+  h[key] = File.read(path)
+end.freeze
+
+# ---------------------------------------------------------------------------
+# Store into a standalone DocumentStore
+# ---------------------------------------------------------------------------
+store = RobotLab::DocumentStore.new
+
+print "Storing #{DOCUMENTS.size} documents... "
+DOCUMENTS.each { |key, text| store.store(key, text) }
+puts "done"
+puts
+DOCUMENTS.each { |key, text| puts "  #{key.to_s.ljust(24)} #{text.split.size} words" }
+puts
+
+# ---------------------------------------------------------------------------
+# Queries — each phrased differently from the document content
+# ---------------------------------------------------------------------------
+QUERIES = [
+  {
+    label: "Database query performance",
+    query: "Why is my Postgres query slow and how do I investigate it?",
+    want:  :postgres_runbook
+  },
+  {
+    label: "Background job failures during outage",
+    query: "Jobs keep failing when Stripe is down. How do I stop them piling up?",
+    want:  :sidekiq_guide
+  },
+  {
+    label: "API breaking changes policy",
+    query: "Can I rename a response field in the API without breaking clients?",
+    want:  :api_versioning_adr
+  },
+  {
+    label: "Cache expiry and memory pressure",
+    query: "Redis is evicting keys unexpectedly and the cache hit rate has dropped.",
+    want:  :redis_caching_guide
+  },
+  {
+    label: "Production outage from table lock",
+    query: "We had an outage caused by a database lock during a migration. What happened?",
+    want:  :incident_postmortem
+  },
+  {
+    label: "Semantic gap — no shared keywords",
+    query: "Connection pool is full and new requests are being rejected.",
+    want:  :postgres_runbook
+  },
+].freeze
+
+QUERIES.each do |q|
+  results = store.search(q[:query], limit: 3)
+  top     = results.first
+  verdict = top[:key] == q[:want] ? "✓ correct" : "✗ expected #{q[:want]}"
+
+  puts "── #{q[:label]}"
+  puts "   Query:      \"#{q[:query]}\""
+  puts "   Top result: #{top[:key]} (#{format("%.3f", top[:score])}) — #{verdict}"
+  puts "   Ranking:    " + results.map { |r| "#{r[:key]} #{format("%.3f", r[:score])}" }.join(" | ")
+  puts
+end
+
+# ---------------------------------------------------------------------------
+# Delete and verify
+# ---------------------------------------------------------------------------
+puts "── Delete :redis_caching_guide, re-run cache query"
+store.delete(:redis_caching_guide)
+results = store.search("Redis evicting keys unexpectedly", limit: 2)
+puts "   Remaining keys: #{store.keys.inspect}"
+puts "   Top result after deletion: #{results.first[:key]}"
+puts
+
+# ---------------------------------------------------------------------------
+# Memory integration
+# ---------------------------------------------------------------------------
+puts "── Memory integration"
+memory = RobotLab::Memory.new(enable_cache: false)
+
+DOCUMENTS.each { |key, text| memory.store_document(key, text) }
+puts "   Stored #{memory.document_keys.size} documents via memory.store_document"
+
+hits = memory.search_documents("slow query bloat vacuum autovacuum", limit: 2)
+puts "   Search 'slow query bloat vacuum autovacuum':"
+hits.each { |h| puts "     #{h[:key]} (#{format("%.3f", h[:score])})" }
+
+memory.delete_document(:postgres_runbook)
+puts "   After delete, keys: #{memory.document_keys.inspect}"
+puts
+
+# ---------------------------------------------------------------------------
+# RAG pattern
+# ---------------------------------------------------------------------------
+puts "=" * 60
+puts "RAG Pattern: retrieve relevant docs, then generate with LLM"
+puts "=" * 60
+puts
+
+rag_query = "Our Sidekiq jobs exhaust retries and land in the dead queue after a Stripe outage."
+
+hits    = store.search(rag_query, limit: 2)
+context = hits.map { |h| h[:text] }.join("\n\n---\n\n")
+
+puts "User question:"
+puts "  \"#{rag_query}\""
+puts
+puts "Retrieved #{hits.size} document(s) — #{context.split.size} words of context:"
+hits.each { |h| puts "  #{h[:key]} (score #{format("%.3f", h[:score])})" }
+puts
+puts "LLM call would be:"
+puts '  robot.run("Use the following docs:\n#{context}\n\nQuestion: #{rag_query}")'
+puts
+
+puts "Done."

data/lib/robot_lab/document_store/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module RobotLab
+  class DocumentStore
+    VERSION = "0.1.0"
+  end
+end

data/lib/robot_lab/document_store.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require "fastembed"
+require_relative "document_store/version"
+
+module RobotLab
+  # Embedding-based document store for semantic search over arbitrary text.
+  #
+  # Documents are embedded using fastembed (BAAI/bge-small-en-v1.5 by default)
+  # and stored in memory. Queries are embedded the same way, then compared by
+  # cosine similarity to find the closest documents.
+  #
+  # The embedding model is initialised lazily on first use — the ONNX model
+  # file is downloaded on that first call (cached locally afterwards).
+  #
+  # @example Standalone
+  #   store = RobotLab::DocumentStore.new
+  #   store.store(:q4_report, "Q4 revenue came in at $4.2M, up 18% YoY…")
+  #   store.store(:q3_report, "Q3 showed 15% growth, driven by APAC…")
+  #
+  #   results = store.search("revenue growth", limit: 2)
+  #   results.each { |r| puts "#{r[:key]} (#{r[:score].round(3)}): #{r[:text][0..60]}" }
+  #
+  # @example With robot_lab Memory
+  #   memory.store_document(:readme, File.read("README.md"))
+  #   memory.search_documents("how to configure redis", limit: 3)
+  #
+  class DocumentStore
+    # Default embedding model used when none is specified.
+    DEFAULT_MODEL = "BAAI/bge-small-en-v1.5"
+
+    # @param model_name [String] fastembed model name
+    def initialize(model_name: DEFAULT_MODEL)
+      @model_name = model_name
+      @documents  = {}  # key (Symbol) => { text: String, vector: Array<Float> }
+      @mutex      = Mutex.new
+      @model      = nil  # lazy: initialised on first embed call
+    end
+
+    # Embed +text+ and store it under +key+.
+    #
+    # If a document already exists under +key+ it is replaced.
+    #
+    # @param key [Symbol, String] identifier for this document
+    # @param text [String] the document text to embed and store
+    # @return [self]
+    def store(key, text)
+      key    = key.to_sym
+      vector = passage_vector(text)
+      @mutex.synchronize { @documents[key] = { text: text, vector: vector } }
+      self
+    end
+
+    # Search for documents semantically similar to +query+.
+    #
+    # @param query [String] natural-language search query
+    # @param limit [Integer] maximum number of results (default 5)
+    # @return [Array<Hash>] results sorted by score descending.
+    #   Each hash contains +:key+, +:text+, and +:score+ (Float 0.0..1.0).
+    def search(query, limit: 5)
+      return [] if empty?
+
+      query_vec = query_vector(query)
+      results   = []
+
+      @mutex.synchronize do
+        @documents.each do |key, doc|
+          score = cosine_similarity(query_vec, doc[:vector])
+          results << { key: key, text: doc[:text], score: score }
+        end
+      end
+
+      results.sort_by { |r| -r[:score] }.first(limit)
+    end
+
+    # Number of stored documents.
+    # @return [Integer]
+    def size
+      @mutex.synchronize { @documents.size }
+    end
+
+    # Keys of all stored documents.
+    # @return [Array<Symbol>]
+    def keys
+      @mutex.synchronize { @documents.keys }
+    end
+
+    # Whether the store contains no documents.
+    # @return [Boolean]
+    def empty?
+      @mutex.synchronize { @documents.empty? }
+    end
+
+    # Remove the document stored under +key+.
+    # @param key [Symbol, String]
+    # @return [self]
+    def delete(key)
+      @mutex.synchronize { @documents.delete(key.to_sym) }
+      self
+    end
+
+    # Remove all stored documents.
+    # @return [self]
+    def clear
+      @mutex.synchronize { @documents.clear }
+      self
+    end
+
+    private
+
+    def model
+      @model ||= Fastembed::TextEmbedding.new(model_name: @model_name, show_progress: false)
+    end
+
+    def passage_vector(text)
+      model.passage_embed([text]).to_a.first
+    end
+
+    def query_vector(text)
+      model.query_embed([text]).to_a.first
+    end
+
+    def cosine_similarity(vec_a, vec_b)
+      return 0.0 unless vec_a && vec_b
+      return 0.0 if vec_a.empty? || vec_b.empty?
+      return 0.0 if vec_a.length != vec_b.length
+
+      dot    = 0.0
+      norm_a = 0.0
+      norm_b = 0.0
+
+      vec_a.each_with_index do |a, i|
+        b       = vec_b[i]
+        dot    += a * b
+        norm_a += a * a
+        norm_b += b * b
+      end
+
+      return 0.0 if norm_a.zero? || norm_b.zero?
+
+      dot / (Math.sqrt(norm_a) * Math.sqrt(norm_b))
+    end
+  end
+end

data/mkdocs.yml

@@ -0,0 +1,113 @@
+site_name: robot_lab-document_store
+site_description: Embedding-based semantic document search for the RobotLab LLM agent framework
+site_author: Dewayne VanHoozer
+site_url: https://madbomber.github.io/robot_lab-document_store
+copyright: Copyright © 2025 Dewayne VanHoozer
+
+repo_name: MadBomber/robot_lab-document_store
+repo_url: https://github.com/MadBomber/robot_lab-document_store
+edit_uri: edit/main/docs/
+
+theme:
+  name: material
+
+  palette:
+    - scheme: default
+      primary: blue
+      accent: amber
+      toggle:
+        icon: material/brightness-7
+        name: Switch to dark mode
+
+    - scheme: slate
+      primary: blue
+      accent: amber
+      toggle:
+        icon: material/brightness-4
+        name: Switch to light mode
+
+  font:
+    text: Roboto
+    code: Roboto Mono
+
+  icon:
+    repo: fontawesome/brands/github
+    logo: material/database-search
+
+  features:
+    - navigation.instant
+    - navigation.tracking
+    - navigation.tabs
+    - navigation.tabs.sticky
+    - navigation.path
+    - navigation.indexes
+    - navigation.top
+    - navigation.footer
+    - toc.follow
+    - search.suggest
+    - search.highlight
+    - search.share
+    - header.autohide
+    - content.code.copy
+    - content.code.annotate
+    - content.tabs.link
+    - content.tooltips
+    - content.action.edit
+    - content.action.view
+
+plugins:
+  - search:
+      separator: '[\s\-,:!=\[\]()"`/]+|\.(?!\d)|&[lg]t;|(?!\b)(?=[A-Z][a-z])'
+
+markdown_extensions:
+  - abbr
+  - admonition
+  - attr_list
+  - def_list
+  - footnotes
+  - md_in_html
+  - tables
+  - toc:
+      permalink: true
+      title: On this page
+  - pymdownx.betterem:
+      smart_enable: all
+  - pymdownx.caret
+  - pymdownx.details
+  - pymdownx.emoji:
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+  - pymdownx.highlight:
+      anchor_linenums: true
+      line_spans: __span
+      pygments_lang_class: true
+  - pymdownx.inlinehilite
+  - pymdownx.magiclink:
+      repo_url_shorthand: true
+      user: MadBomber
+      repo: robot_lab-document_store
+      normalize_issue_symbols: true
+  - pymdownx.mark
+  - pymdownx.smartsymbols
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
+  - pymdownx.tabbed:
+      alternate_style: true
+  - pymdownx.tasklist:
+      custom_checkbox: true
+  - pymdownx.tilde
+
+extra:
+  social:
+    - icon: fontawesome/brands/github
+      link: https://github.com/MadBomber/robot_lab-document_store
+      name: robot_lab-document_store on GitHub
+    - icon: fontawesome/solid/gem
+      link: https://rubygems.org/gems/robot_lab-document_store
+      name: robot_lab-document_store on RubyGems
+
+nav:
+  - Home: index.md

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification
+name: robot_lab-document_store
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dewayne VanHoozer
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fastembed
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Provides RobotLab::DocumentStore — a thread-safe, in-memory semantic
+  search store backed by fastembed (BAAI/bge-small-en-v1.5). Store text documents
+  by key and retrieve the closest matches to a natural-language query using cosine
+  similarity. Works standalone or as a drop-in extension for robot_lab agents and
+  networks.
+email:
+- dvanhoozer@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".envrc"
+- ".github/workflows/deploy-github-pages.yml"
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- docs/index.md
+- examples/26_document_store.rb
+- examples/26_document_store/api_versioning_adr.md
+- examples/26_document_store/incident_postmortem.md
+- examples/26_document_store/postgres_runbook.md
+- examples/26_document_store/redis_caching_guide.md
+- examples/26_document_store/sidekiq_guide.md
+- lib/robot_lab/document_store.rb
+- lib/robot_lab/document_store/version.rb
+- mkdocs.yml
+homepage: https://github.com/madbomber/robot_lab-document_store
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/madbomber/robot_lab-document_store
+  source_code_uri: https://github.com/madbomber/robot_lab-document_store
+  changelog_uri: https://github.com/madbomber/robot_lab-document_store/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.11
+specification_version: 4
+summary: Embedding-based semantic document store for RobotLab agents
+test_files: []