simple-rag-zc 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8b0204bb64f55c075ecb1287b983fda160b1ffcf7e552e346372ab7f95bb3b8
-  data.tar.gz: 330043e72800a113dcc4df223dbfb978c26449ca2c2d477b13db82c7e5c2e743
+  metadata.gz: 906d584b90596bde4fef5efef3f82cceb300284705d6c33023f84dff903f4d2e
+  data.tar.gz: 8d1c292cefc14246e918e06d44cdef48bf31548fd5f1aeaa375b527ee4603458
 SHA512:
-  metadata.gz: 074af0f36149c2e9d5c0b7cd0dacf369d1c21bb4f812bd3c63f4b41772b5a6cb05f3ff8f75f9ed3a5d28f5c9846adb45e11ec0d1b18b7ada77f3a845ca5a989f
-  data.tar.gz: 6c34c79345703bc0cfb83bff5373b9d04a03bbba1c5549a96749b26986b2e49ba56d2e7dc63ba1bf749a0df01a9aec0ac66cb96f5ec03ed1c822a3c73a4379fc
+  metadata.gz: 94d4c13cd41807bf416882f8241f2ea103d00bacf6662b9b901c92a2f4e65463bdae64d204d955850fcfa9703f36ade459a5d23df837be430e4b7373fa70aa1a
+  data.tar.gz: 79f7b78fa5f363b0c5c430f6a4d2af78485cf68b533e26006ef1548a535397a5ffed7e8f19f4d7a6602de1f9ccfdfe73c0f50687fe279796b4ab53d271ad7123

data/README.md

@@ -18,11 +18,11 @@ To release a new version to [RubyGems](https://rubygems.org), run:
 
 ```bash
 gem build simple-rag.gemspec
-gem push simple-rag-$(ruby -Ilib -e 'require "simple_rag/version"; puts SimpleRag::VERSION').gem
+gem push simple-rag-zc-$(ruby -Ilib -e 'require "simple_rag/version"; puts SimpleRag::VERSION').gem
 ```
 
 Install the gem directly:
 
 ```bash
-gem install simple-rag
+gem install simple-rag-zc
 ```

data/example_config.json

@@ -15,7 +15,10 @@
             "reader": "text",
             "threshold": 0.3,
             "dir": "D:\\Studies\\tmp\\learning",
-            "out": "D:\\Studies\\tmp\\learning-gpt1.dt"
+            "out": "D:\\Studies\\tmp\\learning\\learning.dt",
+            "nameMatch": "talks-*.md",
+            "url": "",
+            "searchDefault": false
         }
     ]
-}
+}

data/exe/public/q.html

@@ -114,7 +114,7 @@
                         checkbox.type = 'checkbox';
                         checkbox.id = item.name;
                         checkbox.name = item.name;
-                        checkbox.checked = true;
+                        checkbox.checked = !!item.searchDefault;
 
                         const label = document.createElement('label');
                         label.htmlFor = item.name;
@@ -228,75 +228,6 @@
                 .catch(error => console.error('Error performing agent search:', error));
             }
 
-            function performAgentSearch() {
-                const query = searchInput.value;
-                const configExperiment = configExperimentCheckbox.checked
-                const checkedPaths = Array.from(pathsList.querySelectorAll('input[type="checkbox"]:checked'))
-                    .map(checkbox => checkbox.name);
-
-                fetch('http://localhost:4567/q_plus', {
-                    method: 'POST',
-                    headers: {
-                        'Content-Type': 'application/json',
-                    },
-                    body: JSON.stringify({
-                        q: query,
-                        paths: checkedPaths,
-                        experiment: configExperiment,
-                    })
-                })
-                .then(response => response.json())
-                .then(resp => {
-                    responseContainer.innerHTML = '';
-
-                    if (!!resp.expanded) {
-                        const div = document.createElement('div');
-                        div.className = 'response-item';
-                        div.style.backgroundColor = textToLightColor("expanded");
-                        div.innerHTML = `<div><strong>Expanded Query:</strong> ${resp.expanded}</div>`;
-                        responseContainer.appendChild(div);
-                    }
-
-                    if (resp.variants && resp.variants.length > 0) {
-                        const div = document.createElement('div');
-                        div.className = 'response-item';
-                        div.style.backgroundColor = textToLightColor("variants");
-                        div.innerHTML = `
-                            <div><strong>Variants:</strong> ${resp.variants.join(', ')}</div>
-                        `;
-                        responseContainer.appendChild(div);
-                    }
-
-                    if (!!resp.eval) {
-                        const div = document.createElement('div');
-                        div.className = 'response-item';
-                        div.style.backgroundColor = textToLightColor("experiment");
-                        div.innerHTML = `
-                            <div class="markdown-content">${marked.parse(resp.eval)}</div>
-                        `;
-                        responseContainer.appendChild(div);
-                    }
-
-                    resp.data.forEach(item => {
-                        const div = document.createElement('div');
-                        div.className = 'response-item';
-                        div.style.backgroundColor = textToLightColor(item.lookup);
-                        div.dataset.note = item.text;
-                        div.innerHTML = `
-                            <div><strong>Path:</strong> <a href="${item.url}">${item.id}</a></div>
-                            <div><strong>Score:</strong> ${item.score}</div>
-                            <div class="markdown-content">${marked.parse(item.text)}</div>
-                        `;
-                        const btn = document.createElement('button');
-                        btn.className = 'discuss-button';
-                        btn.textContent = 'Discuss';
-                        btn.addEventListener('click', () => discussCard(div));
-                        div.appendChild(btn);
-                        responseContainer.appendChild(div);
-                    });
-                })
-                .catch(error => console.error('Error performing agent search:', error));
-            }
 
             function textToLightColor(text) {
                 // Generate a hash from the text

data/exe/public/setup.html

@@ -0,0 +1,136 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Setup SimpleRag</title>
+    <style>
+        body { font-family: Arial, sans-serif; margin: 20px; }
+        .path-item { margin-bottom: 20px; padding: 15px; border: 1px solid #ccc; }
+        .path-item input[type="text"], .path-item select { width: 300px; margin-bottom: 10px; }
+        .path-item label { display: block; margin-bottom: 5px; }
+    </style>
+</head>
+<body>
+    <h1>Setup SimpleRag Config</h1>
+    <form id="config-form">
+        <h2>Paths</h2>
+        <div id="paths"></div>
+        <button type="button" onclick="addPath()">Add Path</button>
+        <h2>Chat</h2>
+        <label>Provider: <input id="chat_provider" value="openai"></label><br>
+        <label>URL: <input id="chat_url" value=""></label><br>
+        <label>Model: <input id="chat_model" value="gpt-3.5-turbo-16k"></label>
+        <h2>Embedding</h2>
+        <label>Provider: <input id="emb_provider" value="openai"></label><br>
+        <label>URL: <input id="emb_url" value=""></label><br>
+        <label>Model: <input id="emb_model" value="text-embedding-3-small"></label>
+        <br><br>
+        <button type="submit">Save</button>
+    </form>
+    <script>
+    let READERS = [];
+
+    function fillReaderSelect(select, value){
+        select.innerHTML = READERS.map(r=>`<option value="${r}">${r}</option>`).join('');
+        if(value){ select.value = value; }
+    }
+
+    function createPathDiv(p){
+        const idx = document.querySelectorAll('.path-item').length;
+        const div = document.createElement('div');
+        div.className = 'path-item';
+        div.innerHTML = `
+            <label>Dir: <input type="text" class="pdir" id="dir_${idx}" value="${p?.dir||''}">
+                <input type="file" webkitdirectory directory style="display:none" id="dirsel_${idx}">
+                <button type="button" onclick="document.getElementById('dirsel_${idx}').click()">Select Folder</button></label>
+            <label>Name: <input type="text" class="pname" value="${p?.name||''}"></label>
+            <label>Reader: <select class="preader"></select></label>
+            <label>Threshold: <input type="text" class="pthreshold" value="${p?.threshold||0.3}"></label>
+            <label>Out: <input type="text" class="pout" value="${p?.out||''}"></label>
+            <label>NameMatch: <input type="text" class="pnamematch" value="${p?.nameMatch||''}"></label>
+            <label>URL: <input type="text" class="purl" value="${p?.url||''}"></label>
+            <label>Search Default: <input type="checkbox" class="psearchdefault" ${p?.searchDefault?'checked':''}></label>
+            <button type="button" onclick="this.parentNode.remove()">Remove</button>
+        `;
+        const dirInput = div.querySelector('#dir_'+idx);
+        const nameInput = div.querySelector('.pname');
+        const outInput = div.querySelector('.pout');
+
+        function updateNameOut(){
+            if(!dirInput.value) return;
+            const parts = dirInput.value.replace(/\\/g,'/').split('/').filter(Boolean);
+            const name = parts[parts.length-1] || '';
+            nameInput.value = name;
+            outInput.value = dirInput.value.replace(/[/\\]$/, '') + '/' + name + '.dt';
+        }
+
+        div.querySelector('#dirsel_'+idx).addEventListener('change', function(){
+            if(this.files.length>0){
+                const rel = this.files[0].webkitRelativePath;
+                const dir = rel.split('/')[0];
+                dirInput.value = dir;
+                updateNameOut();
+            }
+        });
+        dirInput.addEventListener('change', updateNameOut);
+        fillReaderSelect(div.querySelector('.preader'), p?.reader||'text');
+        return div;
+    }
+
+    function addPath(p){
+        document.getElementById('paths').appendChild(createPathDiv(p));
+    }
+
+    function loadConfig(readers){
+        READERS = readers;
+        fetch('/config').then(r=>r.json()).then(cfg=>{
+            if(cfg.chat){
+                document.getElementById('chat_provider').value = cfg.chat.provider||'openai';
+                document.getElementById('chat_url').value = cfg.chat.url||'';
+                document.getElementById('chat_model').value = cfg.chat.model||'gpt-3.5-turbo-16k';
+            }
+            if(cfg.embedding){
+                document.getElementById('emb_provider').value = cfg.embedding.provider||'openai';
+                document.getElementById('emb_url').value = cfg.embedding.url||'';
+                document.getElementById('emb_model').value = cfg.embedding.model||'text-embedding-3-small';
+            }
+            if(cfg.paths && cfg.paths.length>0){
+                cfg.paths.forEach(p=>addPath(p));
+            }else{
+                addPath();
+            }
+        });
+    }
+
+    fetch('/readers').then(r=>r.json()).then(loadConfig);
+
+    document.getElementById('config-form').addEventListener('submit', function(e){
+        e.preventDefault();
+        const paths=[];
+        document.querySelectorAll('.path-item').forEach(div=>{
+            paths.push({
+                dir: div.querySelector('.pdir').value,
+                name: div.querySelector('.pname').value,
+                reader: div.querySelector('.preader').value,
+                threshold: parseFloat(div.querySelector('.pthreshold').value)||0,
+                out: div.querySelector('.pout').value,
+                nameMatch: div.querySelector('.pnamematch').value,
+                url: div.querySelector('.purl').value,
+                searchDefault: div.querySelector('.psearchdefault').checked
+            });
+        });
+        const config={
+            chat:{provider:document.getElementById('chat_provider').value,
+                  url:document.getElementById('chat_url').value,
+                  model:document.getElementById('chat_model').value},
+            embedding:{provider:document.getElementById('emb_provider').value,
+                       url:document.getElementById('emb_url').value,
+                       model:document.getElementById('emb_model').value},
+            paths:paths
+        };
+        fetch('/save',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(config)})
+            .then(()=>alert('Saved'));
+    });
+    </script>
+</body>
+</html>

data/exe/run-index

@@ -11,8 +11,7 @@ require "json"
 require "ostruct"
 require "digest"
 
-require_relative "../llm/openai"
-require_relative "../llm/embedding"
+require_relative "../llm/llm"
 require_relative "../readers/reader"
 
 if ARGV.length != 1

data/exe/run-server

@@ -9,7 +9,7 @@
 
 require "json"
 require "ostruct"
-require "sinatra"
+require 'sinatra/base'
 
 require_relative "../server/retriever"
 require_relative "../server/synthesizer"
@@ -23,6 +23,7 @@ end
 config = JSON.parse(File.read(ARGV[0]))
 CONFIG = OpenStruct.new(config)
 CONFIG.paths = CONFIG.paths.map { |p| OpenStruct.new(p) }
+CONFIG.paths.each { |p| p.searchDefault = !!p.searchDefault }
 CONFIG.path_map = {}
 CONFIG.paths.each { |p| CONFIG.path_map[p.name] = p }
 
@@ -32,33 +33,92 @@ if OPENAI_KEY.empty?
     exit 9
 end
 
-# list all the paths that can be searched
-get '/paths' do
-    content_type :json
+class SimpleRagServer < Sinatra::Application
+    # list all the paths that can be searched
+    get '/paths' do
+        content_type :json
 
-    resp = []
-    CONFIG.paths.each do |p|
-        resp << { "name": p.name }
+        resp = []
+        CONFIG.paths.each do |p|
+            resp << { name: p.name, searchDefault: p.searchDefault }
+        end
+        resp.to_json
     end
-    resp.to_json
-end
 
-# query within the paths
-post '/q' do
-    content_type :json
+    # query within the paths
+    post '/q' do
+        content_type :json
+
+        data = JSON.parse(request.body.read)
 
-    data = JSON.parse(request.body.read)
+        selected = data["paths"]
+        if !selected || selected.empty?
+            selected = CONFIG.paths.select { |p| p.searchDefault }.map(&:name)
+            selected = CONFIG.path_map.keys if selected.empty?
+        end
+        lookup_paths = selected.map { |name| CONFIG.path_map[name] }
+
+        topN = (data["topN"] || 20).to_i
+
+        q = data["q"]
+        entries = retrieve_by_embedding(lookup_paths, q)
+        if q.to_s.strip.length < 5 && q.to_s.split(/\s+/).length < 5
+            entries.concat(retrieve_by_text(lookup_paths, q))
+
+            unique = {}
+            entries.each do |e|
+                key = [e["path"], e["chunk"]]
+                if unique[key]
+                    unique[key]["score"] = (unique[key]["score"] || 0) + (e["score"] || 0)
+                else
+                    unique[key] = e
+                end
+            end
+
+            entries = unique.values
+        end
+        entries = entries.sort_by { |item| -item["score"] }.take(topN)
+
+        resp = {
+            data: [],
+        }
 
-    lookup_paths = (data["paths"] || CONFIG.paths_map.keys).map do |name|
-        CONFIG.path_map[name]
+        entries.each do |item|
+            resp[:data] << {
+                path: item["path"],
+                lookup: item["lookup"],
+                id: item["id"],
+                url: item["url"],
+                text: item["reader"].load.get_chunk(item["chunk"]),
+                score: item["score"],
+            }
+        end
+
+        resp.to_json
     end
 
-    topN = (data["topN"] || 20).to_i
+    # agentic query - expand the query using LLM before searching
+    post '/q_plus' do
+        content_type :json
+
+        data = JSON.parse(request.body.read)
+
+        selected = data["paths"]
+        if !selected || selected.empty?
+            selected = CONFIG.paths.select { |p| p.searchDefault }.map(&:name)
+            selected = CONFIG.path_map.keys if selected.empty?
+        end
+        lookup_paths = selected.map { |name| CONFIG.path_map[name] }
+
+        topN = (data["topN"] || 20).to_i
 
-    q = data["q"]
-    entries = retrieve_by_embedding(lookup_paths, q)
-    if q.to_s.strip.length < 5 && q.to_s.split(/\s+/).length < 5
-        entries.concat(retrieve_by_text(lookup_paths, q))
+        expanded_q = expand_query(data["q"])
+        variants = expand_variants(data["q"])
+
+        entries = []
+        entries.concat(retrieve_by_embedding(lookup_paths, data["q"]))
+        entries.concat(retrieve_by_embedding(lookup_paths, expanded_q))
+        variants.each { |v| entries.concat(retrieve_by_text(lookup_paths, v)) }
 
         unique = {}
         entries.each do |e|
@@ -70,98 +130,49 @@ post '/q' do
             end
         end
 
-        entries = unique.values
-    end
-    entries = entries.sort_by { |item| -item["score"] }.take(topN)
-
-    resp = {
-        data: [],
-    }
-
-    entries.each do |item|
-        resp[:data] << {
-            path: item["path"],
-            lookup: item["lookup"],
-            id: item["id"],
-            url: item["url"],
-            text: item["reader"].load.get_chunk(item["chunk"]),
-            score: item["score"],
-        }
-    end
-
-    resp.to_json
-end
+        ordered = unique.values.sort_by { |item| -item["score"] }.take(topN)
 
-# agentic query - expand the query using LLM before searching
-post '/q_plus' do
-    content_type :json
+        resp = {
+            data: [],
+            expanded: expanded_q,
+            variants: variants,
+        }
 
-    data = JSON.parse(request.body.read)
+        ordered.each do |item|
+            resp[:data] << {
+                path: item["path"],
+                lookup: item["lookup"],
+                id: item["id"],
+                url: item["url"],
+                text: item["reader"].load.get_chunk(item["chunk"]),
+                score: item["score"],
+            }
+        end
 
-    lookup_paths = (data["paths"] || CONFIG.paths_map.keys).map do |name|
-        CONFIG.path_map[name]
+        resp.to_json
     end
 
-    topN = (data["topN"] || 20).to_i
+    # synthesize notes into a summary
+    post '/synthesize' do
+        content_type :json
 
-    expanded_q = expand_query(data["q"])
-    variants = expand_variants(data["q"])
+        data = JSON.parse(request.body.read)
 
-    entries = []
-    entries.concat(retrieve_by_embedding(lookup_paths, data["q"]))
-    entries.concat(retrieve_by_embedding(lookup_paths, expanded_q))
-    variants.each { |v| entries.concat(retrieve_by_text(lookup_paths, v)) }
-
-    unique = {}
-    entries.each do |e|
-        key = [e["path"], e["chunk"]]
-        if unique[key]
-            unique[key]["score"] = (unique[key]["score"] || 0) + (e["score"] || 0)
-        else
-            unique[key] = e
-        end
-    end
+        summary = synthesize_notes(data["notes"])
 
-    ordered = unique.values.sort_by { |item| -item["score"] }.take(topN)
-
-    resp = {
-        data: [],
-        expanded: expanded_q,
-        variants: variants,
-    }
-
-    ordered.each do |item|
-        resp[:data] << {
-            path: item["path"],
-            lookup: item["lookup"],
-            id: item["id"],
-            url: item["url"],
-            text: item["reader"].load.get_chunk(item["chunk"]),
-            score: item["score"],
-        }
+        { note: summary }.to_json
     end
 
-    resp.to_json
-end
-
-# synthesize notes into a summary
-post '/synthesize' do
-    content_type :json
+    # generate discussion for a single note
+    post '/discuss' do
+        content_type :json
 
-    data = JSON.parse(request.body.read)
+        data = JSON.parse(request.body.read)
 
-    summary = synthesize_notes(data["notes"])
+        discussion = discuss_note(data["note"])
 
-    { note: summary }.to_json
+        { discussion: discussion }.to_json
+    end
 end
 
-# generate discussion for a single note
-post '/discuss' do
-    content_type :json
-
-    data = JSON.parse(request.body.read)
-
-    discussion = discuss_note(data["note"])
-
-    { discussion: discussion }.to_json
-end
+SimpleRagServer.run!

data/exe/run-setup

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+# Setup a config JSON interactively via a local web page
+#
+# Usage: run-setup config.json
+
+require "json"
+require 'sinatra/base'
+require_relative '../readers/reader'
+
+if ARGV.length != 1
+  STDOUT << "Invalid arguments received, need a config file\n"
+  exit 1
+end
+
+config_path = File.expand_path(ARGV[0])
+
+class SetupServer < Sinatra::Base
+  set :bind, '0.0.0.0'
+  set :port, 4568
+  set :public_folder, File.expand_path('public', __dir__)
+  set :config_path, nil
+
+  get '/' do
+    send_file File.join(settings.public_folder, 'setup.html')
+  end
+
+  get '/readers' do
+    content_type :json
+    READERS.to_json
+  end
+
+  get '/config' do
+    content_type :json
+
+    if File.exist?(settings.config_path)
+      File.read(settings.config_path)
+    else
+      {}.to_json
+    end
+  end
+
+  post '/save' do
+    content_type :json
+
+    data = JSON.parse(request.body.read)
+    File.write(settings.config_path, JSON.pretty_generate(data))
+
+    { status: 'ok' }.to_json
+  end
+end
+
+SetupServer.set :config_path, config_path
+SetupServer.run!

data/lib/simple_rag/version.rb

@@ -1,3 +1,3 @@
 module SimpleRag
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

data/lib/simple_rag.rb

@@ -8,8 +8,7 @@ $LOAD_PATH.unshift File.expand_path("..", __dir__)
 module SimpleRag
 end
 
-require "llm/openai"
-require "llm/embedding"
+require "llm/llm"
 require "readers/reader"
 require "server/retriever"
 require "server/synthesizer"

data/llm/llm.rb

@@ -0,0 +1,57 @@
+require_relative "openai"
+require_relative "ollama"
+
+ROLE_SYSTEM = "system"
+ROLE_USER = "user"
+ROLE_ASSISTANT = "assistant"
+NEXT_ROLE = ->(role) { role != ROLE_USER ? ROLE_USER : ROLE_ASSISTANT }
+
+# Fetch configuration value with defaults
+# Supports Hash or OpenStruct configuration objects
+
+def cfg(section, key, default)
+  return default unless defined?(CONFIG)
+  sec = CONFIG.send(section) if CONFIG.respond_to?(section)
+  return default unless sec
+
+  if sec.is_a?(Hash)
+    sec.fetch(key, default)
+  elsif sec.respond_to?(key)
+    val = sec.send(key)
+    val.nil? ? default : val
+  else
+    default
+  end
+end
+
+# Route chat requests based on provider configuration
+
+def chat(messages, opts = {})
+  provider = cfg(:chat, 'provider', 'openai').downcase
+  case provider
+  when 'ollama'
+    model = cfg(:chat, 'model', 'llama2')
+    url = cfg(:chat, 'url', 'http://localhost:11434/api/chat')
+    ollama_chat(messages, model, url, opts)
+  else
+    model = cfg(:chat, 'model', 'gpt-4.1-mini')
+    url = cfg(:chat, 'url', 'https://api.openai.com/v1/chat/completions')
+    openai_chat(messages, model, url, opts)
+  end
+end
+
+# Route embedding requests based on provider configuration
+
+def embedding(txts, opts = {})
+  provider = cfg(:embedding, 'provider', 'openai').downcase
+  case provider
+  when 'ollama'
+    model = cfg(:embedding, 'model', 'nomic-embed-text')
+    url = cfg(:embedding, 'url', 'http://localhost:11434/api/embeddings')
+    ollama_embedding(txts, model, url, opts)
+  else
+    model = cfg(:embedding, 'model', 'text-embedding-3-small')
+    url = cfg(:embedding, 'url', 'https://api.openai.com/v1/embeddings')
+    openai_embedding(txts, model, url, opts)
+  end
+end

data/llm/ollama.rb

@@ -1,13 +1,12 @@
 require_relative "http"
 
-def embedding_ollama(txts, opts = {})
+def ollama_embedding(txts, model, url, opts = {})
   data = {
-    "model" => "nomic-embed-text",
+    "model" => model,
     "prompt" => txts
   }.merge(opts)
 
-  uri = "http://localhost:11434/api/embeddings"
-  response = http_post(uri, nil, data)
+  response = http_post(url, nil, data)
 
   if response.code != "200"
     STDOUT << "Embedding error: #{response}\n"
@@ -16,4 +15,25 @@ def embedding_ollama(txts, opts = {})
 
   result = JSON.parse(response.body)
   result["embedding"]
+end
+
+def ollama_chat(messages, model, url, opts = {})
+  data = {
+    "model" => model,
+    "messages" => messages
+  }.merge(opts)
+
+  response = http_post(url, nil, data)
+
+  if response.code != "200"
+    STDOUT << "Chat error: #{response}\n"
+    exit 1
+  end
+
+  result = JSON.parse(response.body)
+  if result.is_a?(Hash) && result["message"]
+    result["message"]["content"]
+  else
+    result["choices"][0]["message"]["content"]
+  end
 end

data/llm/openai.rb

@@ -1,18 +1,12 @@
 require_relative "http"
 
-ROLE_SYSTEM = "system"
-ROLE_USER = "user"
-ROLE_ASSISTANT = "assistant"
-NEXT_ROLE = ->(role) { role != ROLE_USER ? ROLE_USER : ROLE_ASSISTANT }
-
-def chat(messages, opts = {})
+def openai_chat(messages, model, url, opts = {})
   data = {
-    "model" => "gpt-4o-mini",
+    "model" => model,
     "messages" => messages
   }.merge(opts)
 
-  uri = "https://api.openai.com/v1/chat/completions"
-  response = http_post(uri, OPENAI_KEY, data)
+  response = http_post(url, OPENAI_KEY, data)
 
   if response.code != "200"
     STDOUT << "Chat error: #{response}\n"
@@ -25,14 +19,13 @@ def chat(messages, opts = {})
   result["choices"][0]["message"]["content"]
 end
 
-def embedding(txts, opts = {})
+def openai_embedding(txts, model, url, opts = {})
   data = {
-    "model" => "text-embedding-3-small",
+    "model" => model,
     "input" => txts
   }.merge(opts)
 
-  uri = "https://api.openai.com/v1/embeddings"
-  response = http_post(uri, OPENAI_KEY, data)
+  response = http_post(url, OPENAI_KEY, data)
 
   if response.code != "200"
     STDOUT << "Embedding error: #{response.body}\n"
@@ -41,4 +34,4 @@ def embedding(txts, opts = {})
 
   result = JSON.parse(response.body)
   result["data"][0]["embedding"]
-end
+end

data/readers/journal.rb

@@ -0,0 +1,69 @@
+class JournalReader
+    SKIP_HEADINGS = ["\u7CBE\u529B", "\u611F\u6069"]
+
+    attr_accessor :file, :chunks
+
+    def initialize(file)
+        @file = file
+        @loaded = false
+        @chunks = []
+    end
+
+    def load
+        return self if @loaded
+
+        parse_journal
+
+        @loaded = true
+        self
+    end
+
+    def get_chunk(idx)
+        @chunks[idx || 0]
+    end
+
+    private
+
+    def parse_journal
+        started = false
+        heading = nil
+        lines = []
+
+        File.foreach(@file) do |line|
+            line = line.chomp
+            next if line.strip.empty?
+
+            if !started
+                next unless line.start_with?("## ")
+                started = true
+                heading = line[3..].strip
+                lines = [clean_line(line)]
+                next
+            end
+
+            if line.start_with?("## ")
+                push_chunk(heading, lines)
+                heading = line[3..].strip
+                lines = [clean_line(line)]
+                next
+            end
+
+            next if line.lstrip.start_with?("<")
+
+            lines << clean_line(line)
+        end
+
+        push_chunk(heading, lines) if started
+    end
+
+    def push_chunk(heading, lines)
+        return if SKIP_HEADINGS.any? { |k| heading.include?(k) }
+        return if lines.length < 3
+
+        @chunks << lines.join("\n")
+    end
+
+    def clean_line(line)
+        line.gsub(/\[([^\]]+)\]\(([^\)]+)\)/, '\\1')
+    end
+end

data/readers/reader.rb

@@ -1,12 +1,21 @@
+READERS = %w[text note journal]
+
 def get_reader(name)
-    case name.downcase
+    case name.to_s.downcase
     when "text"
         require_relative "text"
-        return TextReader
+        TextReader
     when "note"
         require_relative "note"
-        return NoteReader
+        NoteReader
+    when "journal"
+        require_relative "journal"
+        JournalReader
     else
-        return nil
+        nil
     end
-end
+end
+
+def available_readers
+    READERS
+end

data/readers/text.rb

@@ -12,13 +12,26 @@ class TextReader
         return self if @loaded
 
         chunk = ""
+        in_frontmatter = false
         File.foreach(@file) do |line|
-            if line.start_with?(/- .+:/) || line.start_with?('  - [[') # yaml like
+            stripped = line.strip
+
+            if in_frontmatter
+                if stripped == '---' || stripped == '...'
+                    in_frontmatter = false
+                end
+                next
+            elsif stripped == '---'
+                in_frontmatter = true
+                next
+            end
+
+            if line.start_with?('- ') && line.include?(':') || line.start_with?('  - [[')
                 next
-            elsif line.start_with?('<') # html like
+            elsif line.start_with?('<')
                 next
             else
-                chunk << line unless line.strip.empty?
+                chunk << line unless stripped.empty?
             end
         end
 

data/server/discuss.rb

@@ -3,7 +3,7 @@ You provide a short discussion of a note from multiple perspectives.
 Focus on explaining key concepts succinctly.
 PROMPT
 
-require_relative "../llm/openai"
+require_relative "../llm/llm"
 
 # note: string
 # Returns discussion text

data/server/retriever.rb

@@ -1,15 +1,13 @@
 require "pathname"
 
 require_relative "cache"
-
-require_relative "../llm/openai"
+require_relative "../llm/llm"
 require_relative "../llm/embedding"
-
 require_relative "../readers/reader"
 
 AGENT_PROMPT = <<~PROMPT
-You expand a short search query so it is easier to retrieve related markdown
-documents. Return only the expanded query in a single line.
+Expand the user input to a better search query so it is easier to retrieve related markdown
+documents using embedding. Return only the expanded query in a single line.
 PROMPT
 
 def expand_query(q)
@@ -17,7 +15,11 @@ def expand_query(q)
         { role: ROLE_SYSTEM, content: AGENT_PROMPT },
         { role: ROLE_USER, content: q },
     ]
-    chat(msgs).strip
+
+    query = chat(msgs).strip
+    STDOUT << "Expand query: #{query}\n"
+
+    query
 end
 
 def retrieve_by_embedding(lookup_paths, q)
@@ -78,8 +80,8 @@ def extract_url(file_path, url)
 end
 
 VARIANT_PROMPT = <<~PROMPT
-You generate a few alternative short search queries for exact text match.
-Return a JSON array of strings with three different variants.
+Generate three alternative search keywords based on the user input to retrieve related markdown using exact keyword matches.
+Return the search keywords in one CSV line.
 PROMPT
 
 def expand_variants(q)
@@ -87,7 +89,10 @@ def expand_variants(q)
         { role: ROLE_SYSTEM, content: VARIANT_PROMPT },
         { role: ROLE_USER, content: q },
     ]
-    JSON.parse(chat(msgs)) rescue []
+
+    variants = chat(msgs).split(',')
+    STDOUT << "Expand variants: #{variants}\n"
+    variants
 end
 
 def retrieve_by_text(lookup_paths, q)

data/server/synthesizer.rb

@@ -2,7 +2,7 @@ SUM_PROMPT = """You are an expert at combining notes.
 Given a collection of notes, synthesize them into a concise new note capturing the key points.
 """
 
-require_relative "../llm/openai"
+require_relative "../llm/llm"
 
 # notes: array of strings
 # Returns summary text

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: simple-rag-zc
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Zhuochun
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
 date: 2025-06-07 00:00:00.000000000 Z
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: rackup
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: puma
   requirement: !ruby/object:Gem::Requirement
@@ -44,21 +58,26 @@ email:
 executables:
 - run-index
 - run-server
+- run-setup
 extensions: []
 extra_rdoc_files: []
 files:
 - README.md
 - example_config.json
 - exe/public/q.html
+- exe/public/setup.html
 - exe/run-index
 - exe/run-server
+- exe/run-setup
 - lib/simple_rag.rb
 - lib/simple_rag/version.rb
 - llm/embedding.rb
 - llm/http.rb
+- llm/llm.rb
 - llm/ollama.rb
 - llm/openai.rb
 - readers/check-reader.rb
+- readers/journal.rb
 - readers/note.rb
 - readers/reader.rb
 - readers/text.rb
@@ -71,7 +90,7 @@ homepage: https://github.com/zhuochun/simple-rag
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -86,8 +105,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
-signing_key: 
+rubygems_version: 3.4.10
+signing_key:
 specification_version: 4
 summary: RAG on Markdown Files
 test_files: []