simple-rag-zc 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1066044473e95484b129ea587a827a02f3f28adac32251df79ed9f8ee6ebaa08
-  data.tar.gz: 947e9528045e8ac0d0e0845e4649c05020ad17198771453382ad6088f6e81e16
+  metadata.gz: 906d584b90596bde4fef5efef3f82cceb300284705d6c33023f84dff903f4d2e
+  data.tar.gz: 8d1c292cefc14246e918e06d44cdef48bf31548fd5f1aeaa375b527ee4603458
 SHA512:
-  metadata.gz: f301b244bab50ae13b7163dfef9fd884a9b2f25b4a5b3b1b83216221b7955dbfbd82b196b55ffbf224cf8041177276b1426ef11ec6765e43281506ceec902030
-  data.tar.gz: 586ff8501e40e2cbdd7f3f17ae5cbe055946534fc8b115a6a72644d257807e6a8ba6cacc8191b6e53dc7a319ea912137dd0bc0590fc94305eca83052181558e6
+  metadata.gz: 94d4c13cd41807bf416882f8241f2ea103d00bacf6662b9b901c92a2f4e65463bdae64d204d955850fcfa9703f36ade459a5d23df837be430e4b7373fa70aa1a
+  data.tar.gz: 79f7b78fa5f363b0c5c430f6a4d2af78485cf68b533e26006ef1548a535397a5ffed7e8f19f4d7a6602de1f9ccfdfe73c0f50687fe279796b4ab53d271ad7123

data/example_config.json

@@ -15,7 +15,10 @@
             "reader": "text",
             "threshold": 0.3,
             "dir": "D:\\Studies\\tmp\\learning",
-            "out": "D:\\Studies\\tmp\\learning-gpt1.dt"
+            "out": "D:\\Studies\\tmp\\learning\\learning.dt",
+            "nameMatch": "talks-*.md",
+            "url": "",
+            "searchDefault": false
         }
     ]
-}
+}

data/exe/public/q.html

@@ -114,7 +114,7 @@
                         checkbox.type = 'checkbox';
                         checkbox.id = item.name;
                         checkbox.name = item.name;
-                        checkbox.checked = true;
+                        checkbox.checked = !!item.searchDefault;
 
                         const label = document.createElement('label');
                         label.htmlFor = item.name;
@@ -228,63 +228,6 @@
                 .catch(error => console.error('Error performing agent search:', error));
             }
 
-            function performAgentSearch() {
-                const query = searchInput.value;
-                const checkedPaths = Array.from(pathsList.querySelectorAll('input[type="checkbox"]:checked'))
-                    .map(checkbox => checkbox.name);
-
-                fetch('http://localhost:4567/q_plus', {
-                    method: 'POST',
-                    headers: {
-                        'Content-Type': 'application/json',
-                    },
-                    body: JSON.stringify({
-                        q: query,
-                        paths: checkedPaths,
-                    })
-                })
-                .then(response => response.json())
-                .then(resp => {
-                    responseContainer.innerHTML = '';
-
-                    if (!!resp.expanded) {
-                        const div = document.createElement('div');
-                        div.className = 'response-item';
-                        div.style.backgroundColor = textToLightColor("expanded");
-                        div.innerHTML = `<div><strong>Expanded Query:</strong> ${resp.expanded}</div>`;
-                        responseContainer.appendChild(div);
-                    }
-
-                    if (resp.variants && resp.variants.length > 0) {
-                        const div = document.createElement('div');
-                        div.className = 'response-item';
-                        div.style.backgroundColor = textToLightColor("variants");
-                        div.innerHTML = `
-                            <div><strong>Variants:</strong> ${resp.variants.join(', ')}</div>
-                        `;
-                        responseContainer.appendChild(div);
-                    }
-
-                    resp.data.forEach(item => {
-                        const div = document.createElement('div');
-                        div.className = 'response-item';
-                        div.style.backgroundColor = textToLightColor(item.lookup);
-                        div.dataset.note = item.text;
-                        div.innerHTML = `
-                            <div><strong>Path:</strong> <a href="${item.url}">${item.id}</a></div>
-                            <div><strong>Score:</strong> ${item.score}</div>
-                            <div class="markdown-content">${marked.parse(item.text)}</div>
-                        `;
-                        const btn = document.createElement('button');
-                        btn.className = 'discuss-button';
-                        btn.textContent = 'Discuss';
-                        btn.addEventListener('click', () => discussCard(div));
-                        div.appendChild(btn);
-                        responseContainer.appendChild(div);
-                    });
-                })
-                .catch(error => console.error('Error performing agent search:', error));
-            }
 
             function textToLightColor(text) {
                 // Generate a hash from the text

data/exe/public/setup.html

@@ -0,0 +1,136 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Setup SimpleRag</title>
+    <style>
+        body { font-family: Arial, sans-serif; margin: 20px; }
+        .path-item { margin-bottom: 20px; padding: 15px; border: 1px solid #ccc; }
+        .path-item input[type="text"], .path-item select { width: 300px; margin-bottom: 10px; }
+        .path-item label { display: block; margin-bottom: 5px; }
+    </style>
+</head>
+<body>
+    <h1>Setup SimpleRag Config</h1>
+    <form id="config-form">
+        <h2>Paths</h2>
+        <div id="paths"></div>
+        <button type="button" onclick="addPath()">Add Path</button>
+        <h2>Chat</h2>
+        <label>Provider: <input id="chat_provider" value="openai"></label><br>
+        <label>URL: <input id="chat_url" value=""></label><br>
+        <label>Model: <input id="chat_model" value="gpt-3.5-turbo-16k"></label>
+        <h2>Embedding</h2>
+        <label>Provider: <input id="emb_provider" value="openai"></label><br>
+        <label>URL: <input id="emb_url" value=""></label><br>
+        <label>Model: <input id="emb_model" value="text-embedding-3-small"></label>
+        <br><br>
+        <button type="submit">Save</button>
+    </form>
+    <script>
+    let READERS = [];
+
+    function fillReaderSelect(select, value){
+        select.innerHTML = READERS.map(r=>`<option value="${r}">${r}</option>`).join('');
+        if(value){ select.value = value; }
+    }
+
+    function createPathDiv(p){
+        const idx = document.querySelectorAll('.path-item').length;
+        const div = document.createElement('div');
+        div.className = 'path-item';
+        div.innerHTML = `
+            <label>Dir: <input type="text" class="pdir" id="dir_${idx}" value="${p?.dir||''}">
+                <input type="file" webkitdirectory directory style="display:none" id="dirsel_${idx}">
+                <button type="button" onclick="document.getElementById('dirsel_${idx}').click()">Select Folder</button></label>
+            <label>Name: <input type="text" class="pname" value="${p?.name||''}"></label>
+            <label>Reader: <select class="preader"></select></label>
+            <label>Threshold: <input type="text" class="pthreshold" value="${p?.threshold||0.3}"></label>
+            <label>Out: <input type="text" class="pout" value="${p?.out||''}"></label>
+            <label>NameMatch: <input type="text" class="pnamematch" value="${p?.nameMatch||''}"></label>
+            <label>URL: <input type="text" class="purl" value="${p?.url||''}"></label>
+            <label>Search Default: <input type="checkbox" class="psearchdefault" ${p?.searchDefault?'checked':''}></label>
+            <button type="button" onclick="this.parentNode.remove()">Remove</button>
+        `;
+        const dirInput = div.querySelector('#dir_'+idx);
+        const nameInput = div.querySelector('.pname');
+        const outInput = div.querySelector('.pout');
+
+        function updateNameOut(){
+            if(!dirInput.value) return;
+            const parts = dirInput.value.replace(/\\/g,'/').split('/').filter(Boolean);
+            const name = parts[parts.length-1] || '';
+            nameInput.value = name;
+            outInput.value = dirInput.value.replace(/[/\\]$/, '') + '/' + name + '.dt';
+        }
+
+        div.querySelector('#dirsel_'+idx).addEventListener('change', function(){
+            if(this.files.length>0){
+                const rel = this.files[0].webkitRelativePath;
+                const dir = rel.split('/')[0];
+                dirInput.value = dir;
+                updateNameOut();
+            }
+        });
+        dirInput.addEventListener('change', updateNameOut);
+        fillReaderSelect(div.querySelector('.preader'), p?.reader||'text');
+        return div;
+    }
+
+    function addPath(p){
+        document.getElementById('paths').appendChild(createPathDiv(p));
+    }
+
+    function loadConfig(readers){
+        READERS = readers;
+        fetch('/config').then(r=>r.json()).then(cfg=>{
+            if(cfg.chat){
+                document.getElementById('chat_provider').value = cfg.chat.provider||'openai';
+                document.getElementById('chat_url').value = cfg.chat.url||'';
+                document.getElementById('chat_model').value = cfg.chat.model||'gpt-3.5-turbo-16k';
+            }
+            if(cfg.embedding){
+                document.getElementById('emb_provider').value = cfg.embedding.provider||'openai';
+                document.getElementById('emb_url').value = cfg.embedding.url||'';
+                document.getElementById('emb_model').value = cfg.embedding.model||'text-embedding-3-small';
+            }
+            if(cfg.paths && cfg.paths.length>0){
+                cfg.paths.forEach(p=>addPath(p));
+            }else{
+                addPath();
+            }
+        });
+    }
+
+    fetch('/readers').then(r=>r.json()).then(loadConfig);
+
+    document.getElementById('config-form').addEventListener('submit', function(e){
+        e.preventDefault();
+        const paths=[];
+        document.querySelectorAll('.path-item').forEach(div=>{
+            paths.push({
+                dir: div.querySelector('.pdir').value,
+                name: div.querySelector('.pname').value,
+                reader: div.querySelector('.preader').value,
+                threshold: parseFloat(div.querySelector('.pthreshold').value)||0,
+                out: div.querySelector('.pout').value,
+                nameMatch: div.querySelector('.pnamematch').value,
+                url: div.querySelector('.purl').value,
+                searchDefault: div.querySelector('.psearchdefault').checked
+            });
+        });
+        const config={
+            chat:{provider:document.getElementById('chat_provider').value,
+                  url:document.getElementById('chat_url').value,
+                  model:document.getElementById('chat_model').value},
+            embedding:{provider:document.getElementById('emb_provider').value,
+                       url:document.getElementById('emb_url').value,
+                       model:document.getElementById('emb_model').value},
+            paths:paths
+        };
+        fetch('/save',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(config)})
+            .then(()=>alert('Saved'));
+    });
+    </script>
+</body>
+</html>

data/exe/run-index

@@ -11,8 +11,7 @@ require "json"
 require "ostruct"
 require "digest"
 
-require_relative "../llm/openai"
-require_relative "../llm/embedding"
+require_relative "../llm/llm"
 require_relative "../readers/reader"
 
 if ARGV.length != 1

data/exe/run-server

@@ -23,6 +23,7 @@ end
 config = JSON.parse(File.read(ARGV[0]))
 CONFIG = OpenStruct.new(config)
 CONFIG.paths = CONFIG.paths.map { |p| OpenStruct.new(p) }
+CONFIG.paths.each { |p| p.searchDefault = !!p.searchDefault }
 CONFIG.path_map = {}
 CONFIG.paths.each { |p| CONFIG.path_map[p.name] = p }
 
@@ -39,7 +40,7 @@ class SimpleRagServer < Sinatra::Application
 
         resp = []
         CONFIG.paths.each do |p|
-            resp << { "name": p.name }
+            resp << { name: p.name, searchDefault: p.searchDefault }
         end
         resp.to_json
     end
@@ -50,9 +51,12 @@ class SimpleRagServer < Sinatra::Application
 
         data = JSON.parse(request.body.read)
 
-        lookup_paths = (data["paths"] || CONFIG.paths_map.keys).map do |name|
-            CONFIG.path_map[name]
+        selected = data["paths"]
+        if !selected || selected.empty?
+            selected = CONFIG.paths.select { |p| p.searchDefault }.map(&:name)
+            selected = CONFIG.path_map.keys if selected.empty?
         end
+        lookup_paths = selected.map { |name| CONFIG.path_map[name] }
 
         topN = (data["topN"] || 20).to_i
 
@@ -99,9 +103,12 @@ class SimpleRagServer < Sinatra::Application
 
         data = JSON.parse(request.body.read)
 
-        lookup_paths = (data["paths"] || CONFIG.paths_map.keys).map do |name|
-            CONFIG.path_map[name]
+        selected = data["paths"]
+        if !selected || selected.empty?
+            selected = CONFIG.paths.select { |p| p.searchDefault }.map(&:name)
+            selected = CONFIG.path_map.keys if selected.empty?
         end
+        lookup_paths = selected.map { |name| CONFIG.path_map[name] }
 
         topN = (data["topN"] || 20).to_i
 

data/exe/run-setup

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+# Setup a config JSON interactively via a local web page
+#
+# Usage: run-setup config.json
+
+require "json"
+require 'sinatra/base'
+require_relative '../readers/reader'
+
+if ARGV.length != 1
+  STDOUT << "Invalid arguments received, need a config file\n"
+  exit 1
+end
+
+config_path = File.expand_path(ARGV[0])
+
+class SetupServer < Sinatra::Base
+  set :bind, '0.0.0.0'
+  set :port, 4568
+  set :public_folder, File.expand_path('public', __dir__)
+  set :config_path, nil
+
+  get '/' do
+    send_file File.join(settings.public_folder, 'setup.html')
+  end
+
+  get '/readers' do
+    content_type :json
+    READERS.to_json
+  end
+
+  get '/config' do
+    content_type :json
+
+    if File.exist?(settings.config_path)
+      File.read(settings.config_path)
+    else
+      {}.to_json
+    end
+  end
+
+  post '/save' do
+    content_type :json
+
+    data = JSON.parse(request.body.read)
+    File.write(settings.config_path, JSON.pretty_generate(data))
+
+    { status: 'ok' }.to_json
+  end
+end
+
+SetupServer.set :config_path, config_path
+SetupServer.run!

data/lib/simple_rag/version.rb

@@ -1,3 +1,3 @@
 module SimpleRag
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/simple_rag.rb

@@ -8,8 +8,7 @@ $LOAD_PATH.unshift File.expand_path("..", __dir__)
 module SimpleRag
 end
 
-require "llm/openai"
-require "llm/embedding"
+require "llm/llm"
 require "readers/reader"
 require "server/retriever"
 require "server/synthesizer"

data/llm/llm.rb

@@ -0,0 +1,57 @@
+require_relative "openai"
+require_relative "ollama"
+
+ROLE_SYSTEM = "system"
+ROLE_USER = "user"
+ROLE_ASSISTANT = "assistant"
+NEXT_ROLE = ->(role) { role != ROLE_USER ? ROLE_USER : ROLE_ASSISTANT }
+
+# Fetch configuration value with defaults
+# Supports Hash or OpenStruct configuration objects
+
+def cfg(section, key, default)
+  return default unless defined?(CONFIG)
+  sec = CONFIG.send(section) if CONFIG.respond_to?(section)
+  return default unless sec
+
+  if sec.is_a?(Hash)
+    sec.fetch(key, default)
+  elsif sec.respond_to?(key)
+    val = sec.send(key)
+    val.nil? ? default : val
+  else
+    default
+  end
+end
+
+# Route chat requests based on provider configuration
+
+def chat(messages, opts = {})
+  provider = cfg(:chat, 'provider', 'openai').downcase
+  case provider
+  when 'ollama'
+    model = cfg(:chat, 'model', 'llama2')
+    url = cfg(:chat, 'url', 'http://localhost:11434/api/chat')
+    ollama_chat(messages, model, url, opts)
+  else
+    model = cfg(:chat, 'model', 'gpt-4.1-mini')
+    url = cfg(:chat, 'url', 'https://api.openai.com/v1/chat/completions')
+    openai_chat(messages, model, url, opts)
+  end
+end
+
+# Route embedding requests based on provider configuration
+
+def embedding(txts, opts = {})
+  provider = cfg(:embedding, 'provider', 'openai').downcase
+  case provider
+  when 'ollama'
+    model = cfg(:embedding, 'model', 'nomic-embed-text')
+    url = cfg(:embedding, 'url', 'http://localhost:11434/api/embeddings')
+    ollama_embedding(txts, model, url, opts)
+  else
+    model = cfg(:embedding, 'model', 'text-embedding-3-small')
+    url = cfg(:embedding, 'url', 'https://api.openai.com/v1/embeddings')
+    openai_embedding(txts, model, url, opts)
+  end
+end

data/llm/ollama.rb

@@ -1,13 +1,12 @@
 require_relative "http"
 
-def embedding_ollama(txts, opts = {})
+def ollama_embedding(txts, model, url, opts = {})
   data = {
-    "model" => "nomic-embed-text",
+    "model" => model,
     "prompt" => txts
   }.merge(opts)
 
-  uri = "http://localhost:11434/api/embeddings"
-  response = http_post(uri, nil, data)
+  response = http_post(url, nil, data)
 
   if response.code != "200"
     STDOUT << "Embedding error: #{response}\n"
@@ -16,4 +15,25 @@ def embedding_ollama(txts, opts = {})
 
   result = JSON.parse(response.body)
   result["embedding"]
+end
+
+def ollama_chat(messages, model, url, opts = {})
+  data = {
+    "model" => model,
+    "messages" => messages
+  }.merge(opts)
+
+  response = http_post(url, nil, data)
+
+  if response.code != "200"
+    STDOUT << "Chat error: #{response}\n"
+    exit 1
+  end
+
+  result = JSON.parse(response.body)
+  if result.is_a?(Hash) && result["message"]
+    result["message"]["content"]
+  else
+    result["choices"][0]["message"]["content"]
+  end
 end

data/llm/openai.rb

@@ -1,18 +1,12 @@
 require_relative "http"
 
-ROLE_SYSTEM = "system"
-ROLE_USER = "user"
-ROLE_ASSISTANT = "assistant"
-NEXT_ROLE = ->(role) { role != ROLE_USER ? ROLE_USER : ROLE_ASSISTANT }
-
-def chat(messages, opts = {})
+def openai_chat(messages, model, url, opts = {})
   data = {
-    "model" => "gpt-4.1-mini",
+    "model" => model,
     "messages" => messages
   }.merge(opts)
 
-  uri = "https://api.openai.com/v1/chat/completions"
-  response = http_post(uri, OPENAI_KEY, data)
+  response = http_post(url, OPENAI_KEY, data)
 
   if response.code != "200"
     STDOUT << "Chat error: #{response}\n"
@@ -25,14 +19,13 @@ def chat(messages, opts = {})
   result["choices"][0]["message"]["content"]
 end
 
-def embedding(txts, opts = {})
+def openai_embedding(txts, model, url, opts = {})
   data = {
-    "model" => "text-embedding-3-small",
+    "model" => model,
     "input" => txts
   }.merge(opts)
 
-  uri = "https://api.openai.com/v1/embeddings"
-  response = http_post(uri, OPENAI_KEY, data)
+  response = http_post(url, OPENAI_KEY, data)
 
   if response.code != "200"
     STDOUT << "Embedding error: #{response.body}\n"
@@ -41,4 +34,4 @@ def embedding(txts, opts = {})
 
   result = JSON.parse(response.body)
   result["data"][0]["embedding"]
-end
+end

data/readers/journal.rb

@@ -0,0 +1,69 @@
+class JournalReader
+    SKIP_HEADINGS = ["\u7CBE\u529B", "\u611F\u6069"]
+
+    attr_accessor :file, :chunks
+
+    def initialize(file)
+        @file = file
+        @loaded = false
+        @chunks = []
+    end
+
+    def load
+        return self if @loaded
+
+        parse_journal
+
+        @loaded = true
+        self
+    end
+
+    def get_chunk(idx)
+        @chunks[idx || 0]
+    end
+
+    private
+
+    def parse_journal
+        started = false
+        heading = nil
+        lines = []
+
+        File.foreach(@file) do |line|
+            line = line.chomp
+            next if line.strip.empty?
+
+            if !started
+                next unless line.start_with?("## ")
+                started = true
+                heading = line[3..].strip
+                lines = [clean_line(line)]
+                next
+            end
+
+            if line.start_with?("## ")
+                push_chunk(heading, lines)
+                heading = line[3..].strip
+                lines = [clean_line(line)]
+                next
+            end
+
+            next if line.lstrip.start_with?("<")
+
+            lines << clean_line(line)
+        end
+
+        push_chunk(heading, lines) if started
+    end
+
+    def push_chunk(heading, lines)
+        return if SKIP_HEADINGS.any? { |k| heading.include?(k) }
+        return if lines.length < 3
+
+        @chunks << lines.join("\n")
+    end
+
+    def clean_line(line)
+        line.gsub(/\[([^\]]+)\]\(([^\)]+)\)/, '\\1')
+    end
+end

data/readers/reader.rb

@@ -1,12 +1,21 @@
+READERS = %w[text note journal]
+
 def get_reader(name)
-    case name.downcase
+    case name.to_s.downcase
     when "text"
         require_relative "text"
-        return TextReader
+        TextReader
     when "note"
         require_relative "note"
-        return NoteReader
+        NoteReader
+    when "journal"
+        require_relative "journal"
+        JournalReader
     else
-        return nil
+        nil
     end
-end
+end
+
+def available_readers
+    READERS
+end

data/readers/text.rb

@@ -12,13 +12,26 @@ class TextReader
         return self if @loaded
 
         chunk = ""
+        in_frontmatter = false
         File.foreach(@file) do |line|
-            if line.start_with?(/- .+:/) || line.start_with?('  - [[') # yaml like
+            stripped = line.strip
+
+            if in_frontmatter
+                if stripped == '---' || stripped == '...'
+                    in_frontmatter = false
+                end
+                next
+            elsif stripped == '---'
+                in_frontmatter = true
+                next
+            end
+
+            if line.start_with?('- ') && line.include?(':') || line.start_with?('  - [[')
                 next
-            elsif line.start_with?('<') # html like
+            elsif line.start_with?('<')
                 next
             else
-                chunk << line unless line.strip.empty?
+                chunk << line unless stripped.empty?
             end
         end
 

data/server/discuss.rb

@@ -3,7 +3,7 @@ You provide a short discussion of a note from multiple perspectives.
 Focus on explaining key concepts succinctly.
 PROMPT
 
-require_relative "../llm/openai"
+require_relative "../llm/llm"
 
 # note: string
 # Returns discussion text

data/server/retriever.rb

@@ -1,10 +1,8 @@
 require "pathname"
 
 require_relative "cache"
-
-require_relative "../llm/openai"
+require_relative "../llm/llm"
 require_relative "../llm/embedding"
-
 require_relative "../readers/reader"
 
 AGENT_PROMPT = <<~PROMPT

data/server/synthesizer.rb

@@ -2,7 +2,7 @@ SUM_PROMPT = """You are an expert at combining notes.
 Given a collection of notes, synthesize them into a concise new note capturing the key points.
 """
 
-require_relative "../llm/openai"
+require_relative "../llm/llm"
 
 # notes: array of strings
 # Returns summary text

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: simple-rag-zc
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Zhuochun
@@ -58,21 +58,26 @@ email:
 executables:
 - run-index
 - run-server
+- run-setup
 extensions: []
 extra_rdoc_files: []
 files:
 - README.md
 - example_config.json
 - exe/public/q.html
+- exe/public/setup.html
 - exe/run-index
 - exe/run-server
+- exe/run-setup
 - lib/simple_rag.rb
 - lib/simple_rag/version.rb
 - llm/embedding.rb
 - llm/http.rb
+- llm/llm.rb
 - llm/ollama.rb
 - llm/openai.rb
 - readers/check-reader.rb
+- readers/journal.rb
 - readers/note.rb
 - readers/reader.rb
 - readers/text.rb