simple-rag-zc 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8b0204bb64f55c075ecb1287b983fda160b1ffcf7e552e346372ab7f95bb3b8
-  data.tar.gz: 330043e72800a113dcc4df223dbfb978c26449ca2c2d477b13db82c7e5c2e743
+  metadata.gz: 1066044473e95484b129ea587a827a02f3f28adac32251df79ed9f8ee6ebaa08
+  data.tar.gz: 947e9528045e8ac0d0e0845e4649c05020ad17198771453382ad6088f6e81e16
 SHA512:
-  metadata.gz: 074af0f36149c2e9d5c0b7cd0dacf369d1c21bb4f812bd3c63f4b41772b5a6cb05f3ff8f75f9ed3a5d28f5c9846adb45e11ec0d1b18b7ada77f3a845ca5a989f
-  data.tar.gz: 6c34c79345703bc0cfb83bff5373b9d04a03bbba1c5549a96749b26986b2e49ba56d2e7dc63ba1bf749a0df01a9aec0ac66cb96f5ec03ed1c822a3c73a4379fc
+  metadata.gz: f301b244bab50ae13b7163dfef9fd884a9b2f25b4a5b3b1b83216221b7955dbfbd82b196b55ffbf224cf8041177276b1426ef11ec6765e43281506ceec902030
+  data.tar.gz: 586ff8501e40e2cbdd7f3f17ae5cbe055946534fc8b115a6a72644d257807e6a8ba6cacc8191b6e53dc7a319ea912137dd0bc0590fc94305eca83052181558e6

data/README.md

@@ -18,11 +18,11 @@ To release a new version to [RubyGems](https://rubygems.org), run:
 
 ```bash
 gem build simple-rag.gemspec
-gem push simple-rag-$(ruby -Ilib -e 'require "simple_rag/version"; puts SimpleRag::VERSION').gem
+gem push simple-rag-zc-$(ruby -Ilib -e 'require "simple_rag/version"; puts SimpleRag::VERSION').gem
 ```
 
 Install the gem directly:
 
 ```bash
-gem install simple-rag
+gem install simple-rag-zc
 ```

data/exe/public/q.html

@@ -230,7 +230,6 @@
 
             function performAgentSearch() {
                 const query = searchInput.value;
-                const configExperiment = configExperimentCheckbox.checked
                 const checkedPaths = Array.from(pathsList.querySelectorAll('input[type="checkbox"]:checked'))
                     .map(checkbox => checkbox.name);
 
@@ -242,7 +241,6 @@
                     body: JSON.stringify({
                         q: query,
                         paths: checkedPaths,
-                        experiment: configExperiment,
                     })
                 })
                 .then(response => response.json())
@@ -267,16 +265,6 @@
                         responseContainer.appendChild(div);
                     }
 
-                    if (!!resp.eval) {
-                        const div = document.createElement('div');
-                        div.className = 'response-item';
-                        div.style.backgroundColor = textToLightColor("experiment");
-                        div.innerHTML = `
-                            <div class="markdown-content">${marked.parse(resp.eval)}</div>
-                        `;
-                        responseContainer.appendChild(div);
-                    }
-
                     resp.data.forEach(item => {
                         const div = document.createElement('div');
                         div.className = 'response-item';

data/exe/run-server

@@ -9,7 +9,7 @@
 
 require "json"
 require "ostruct"
-require "sinatra"
+require 'sinatra/base'
 
 require_relative "../server/retriever"
 require_relative "../server/synthesizer"
@@ -32,33 +32,86 @@ if OPENAI_KEY.empty?
     exit 9
 end
 
-# list all the paths that can be searched
-get '/paths' do
-    content_type :json
+class SimpleRagServer < Sinatra::Application
+    # list all the paths that can be searched
+    get '/paths' do
+        content_type :json
 
-    resp = []
-    CONFIG.paths.each do |p|
-        resp << { "name": p.name }
+        resp = []
+        CONFIG.paths.each do |p|
+            resp << { "name": p.name }
+        end
+        resp.to_json
     end
-    resp.to_json
-end
 
-# query within the paths
-post '/q' do
-    content_type :json
+    # query within the paths
+    post '/q' do
+        content_type :json
 
-    data = JSON.parse(request.body.read)
+        data = JSON.parse(request.body.read)
 
-    lookup_paths = (data["paths"] || CONFIG.paths_map.keys).map do |name|
-        CONFIG.path_map[name]
+        lookup_paths = (data["paths"] || CONFIG.paths_map.keys).map do |name|
+            CONFIG.path_map[name]
+        end
+
+        topN = (data["topN"] || 20).to_i
+
+        q = data["q"]
+        entries = retrieve_by_embedding(lookup_paths, q)
+        if q.to_s.strip.length < 5 && q.to_s.split(/\s+/).length < 5
+            entries.concat(retrieve_by_text(lookup_paths, q))
+
+            unique = {}
+            entries.each do |e|
+                key = [e["path"], e["chunk"]]
+                if unique[key]
+                    unique[key]["score"] = (unique[key]["score"] || 0) + (e["score"] || 0)
+                else
+                    unique[key] = e
+                end
+            end
+
+            entries = unique.values
+        end
+        entries = entries.sort_by { |item| -item["score"] }.take(topN)
+
+        resp = {
+            data: [],
+        }
+
+        entries.each do |item|
+            resp[:data] << {
+                path: item["path"],
+                lookup: item["lookup"],
+                id: item["id"],
+                url: item["url"],
+                text: item["reader"].load.get_chunk(item["chunk"]),
+                score: item["score"],
+            }
+        end
+
+        resp.to_json
     end
 
-    topN = (data["topN"] || 20).to_i
+    # agentic query - expand the query using LLM before searching
+    post '/q_plus' do
+        content_type :json
+
+        data = JSON.parse(request.body.read)
+
+        lookup_paths = (data["paths"] || CONFIG.paths_map.keys).map do |name|
+            CONFIG.path_map[name]
+        end
+
+        topN = (data["topN"] || 20).to_i
+
+        expanded_q = expand_query(data["q"])
+        variants = expand_variants(data["q"])
 
-    q = data["q"]
-    entries = retrieve_by_embedding(lookup_paths, q)
-    if q.to_s.strip.length < 5 && q.to_s.split(/\s+/).length < 5
-        entries.concat(retrieve_by_text(lookup_paths, q))
+        entries = []
+        entries.concat(retrieve_by_embedding(lookup_paths, data["q"]))
+        entries.concat(retrieve_by_embedding(lookup_paths, expanded_q))
+        variants.each { |v| entries.concat(retrieve_by_text(lookup_paths, v)) }
 
         unique = {}
         entries.each do |e|
@@ -70,98 +123,49 @@ post '/q' do
             end
         end
 
-        entries = unique.values
-    end
-    entries = entries.sort_by { |item| -item["score"] }.take(topN)
-
-    resp = {
-        data: [],
-    }
-
-    entries.each do |item|
-        resp[:data] << {
-            path: item["path"],
-            lookup: item["lookup"],
-            id: item["id"],
-            url: item["url"],
-            text: item["reader"].load.get_chunk(item["chunk"]),
-            score: item["score"],
-        }
-    end
-
-    resp.to_json
-end
+        ordered = unique.values.sort_by { |item| -item["score"] }.take(topN)
 
-# agentic query - expand the query using LLM before searching
-post '/q_plus' do
-    content_type :json
+        resp = {
+            data: [],
+            expanded: expanded_q,
+            variants: variants,
+        }
 
-    data = JSON.parse(request.body.read)
+        ordered.each do |item|
+            resp[:data] << {
+                path: item["path"],
+                lookup: item["lookup"],
+                id: item["id"],
+                url: item["url"],
+                text: item["reader"].load.get_chunk(item["chunk"]),
+                score: item["score"],
+            }
+        end
 
-    lookup_paths = (data["paths"] || CONFIG.paths_map.keys).map do |name|
-        CONFIG.path_map[name]
+        resp.to_json
     end
 
-    topN = (data["topN"] || 20).to_i
+    # synthesize notes into a summary
+    post '/synthesize' do
+        content_type :json
 
-    expanded_q = expand_query(data["q"])
-    variants = expand_variants(data["q"])
+        data = JSON.parse(request.body.read)
 
-    entries = []
-    entries.concat(retrieve_by_embedding(lookup_paths, data["q"]))
-    entries.concat(retrieve_by_embedding(lookup_paths, expanded_q))
-    variants.each { |v| entries.concat(retrieve_by_text(lookup_paths, v)) }
+        summary = synthesize_notes(data["notes"])
 
-    unique = {}
-    entries.each do |e|
-        key = [e["path"], e["chunk"]]
-        if unique[key]
-            unique[key]["score"] = (unique[key]["score"] || 0) + (e["score"] || 0)
-        else
-            unique[key] = e
-        end
+        { note: summary }.to_json
     end
 
-    ordered = unique.values.sort_by { |item| -item["score"] }.take(topN)
-
-    resp = {
-        data: [],
-        expanded: expanded_q,
-        variants: variants,
-    }
-
-    ordered.each do |item|
-        resp[:data] << {
-            path: item["path"],
-            lookup: item["lookup"],
-            id: item["id"],
-            url: item["url"],
-            text: item["reader"].load.get_chunk(item["chunk"]),
-            score: item["score"],
-        }
-    end
-
-    resp.to_json
-end
-
-# synthesize notes into a summary
-post '/synthesize' do
-    content_type :json
+    # generate discussion for a single note
+    post '/discuss' do
+        content_type :json
 
-    data = JSON.parse(request.body.read)
+        data = JSON.parse(request.body.read)
 
-    summary = synthesize_notes(data["notes"])
+        discussion = discuss_note(data["note"])
 
-    { note: summary }.to_json
+        { discussion: discussion }.to_json
+    end
 end
 
-# generate discussion for a single note
-post '/discuss' do
-    content_type :json
-
-    data = JSON.parse(request.body.read)
-
-    discussion = discuss_note(data["note"])
-
-    { discussion: discussion }.to_json
-end
+SimpleRagServer.run!

data/lib/simple_rag/version.rb

@@ -1,3 +1,3 @@
 module SimpleRag
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/llm/openai.rb

@@ -7,7 +7,7 @@ NEXT_ROLE = ->(role) { role != ROLE_USER ? ROLE_USER : ROLE_ASSISTANT }
 
 def chat(messages, opts = {})
   data = {
-    "model" => "gpt-4o-mini",
+    "model" => "gpt-4.1-mini",
     "messages" => messages
   }.merge(opts)
 

data/server/retriever.rb

@@ -8,8 +8,8 @@ require_relative "../llm/embedding"
 require_relative "../readers/reader"
 
 AGENT_PROMPT = <<~PROMPT
-You expand a short search query so it is easier to retrieve related markdown
-documents. Return only the expanded query in a single line.
+Expand the user input to a better search query so it is easier to retrieve related markdown
+documents using embedding. Return only the expanded query in a single line.
 PROMPT
 
 def expand_query(q)
@@ -17,7 +17,11 @@ def expand_query(q)
         { role: ROLE_SYSTEM, content: AGENT_PROMPT },
         { role: ROLE_USER, content: q },
     ]
-    chat(msgs).strip
+
+    query = chat(msgs).strip
+    STDOUT << "Expand query: #{query}\n"
+
+    query
 end
 
 def retrieve_by_embedding(lookup_paths, q)
@@ -78,8 +82,8 @@ def extract_url(file_path, url)
 end
 
 VARIANT_PROMPT = <<~PROMPT
-You generate a few alternative short search queries for exact text match.
-Return a JSON array of strings with three different variants.
+Generate three alternative search keywords based on the user input to retrieve related markdown using exact keyword matches.
+Return the search keywords in one CSV line.
 PROMPT
 
 def expand_variants(q)
@@ -87,7 +91,10 @@ def expand_variants(q)
         { role: ROLE_SYSTEM, content: VARIANT_PROMPT },
         { role: ROLE_USER, content: q },
     ]
-    JSON.parse(chat(msgs)) rescue []
+
+    variants = chat(msgs).split(',')
+    STDOUT << "Expand variants: #{variants}\n"
+    variants
 end
 
 def retrieve_by_text(lookup_paths, q)

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: simple-rag-zc
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Zhuochun
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
 date: 2025-06-07 00:00:00.000000000 Z
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: rackup
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: puma
   requirement: !ruby/object:Gem::Requirement
@@ -71,7 +85,7 @@ homepage: https://github.com/zhuochun/simple-rag
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -86,8 +100,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
-signing_key: 
+rubygems_version: 3.4.10
+signing_key:
 specification_version: 4
 summary: RAG on Markdown Files
 test_files: []