bonecode 1.0.0

package/scripts/test_rag.js

@@ -0,0 +1,189 @@
+/**
+ * BoneCode RAG Integration Test
+ * Tests: embedding, chunk insert, hybrid search, memory search, LLM prompt
+ */
+require("dotenv").config();
+const { Pool } = require("pg");
+const { v4: uuid } = require("uuid");
+
+const pool = new Pool({ connectionString: process.env.DATABASE_URL });
+const BASE = "http://localhost:3000";
+const LLM_BASE = "http://10.0.0.20:1234/v1";
+const EMBED_MODEL = process.env.EMBEDDING_MODEL || "text-embedding-nomic-embed-text-v1.5";
+const CHAT_MODEL = process.env.DEFAULT_MODEL || "qwen3.6-35b-a3b-uncensored-heretic-apex";
+
+const jwt = require("jsonwebtoken");
+const token = jwt.sign({ sub: "test-user" }, process.env.JWT_SECRET, { expiresIn: "1h" });
+const auth = { Authorization: "Bearer " + token, "Content-Type": "application/json" };
+
+let pass = 0, fail = 0;
+const check = (label, ok, detail = "") => {
+  console.log((ok ? "PASS" : "FAIL"), label, detail || "");
+  ok ? pass++ : fail++;
+};
+
+async function embed(text) {
+  const r = await fetch(`${LLM_BASE}/embeddings`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json", Authorization: "Bearer not-needed" },
+    body: JSON.stringify({ model: EMBED_MODEL, input: text }),
+  });
+  const d = await r.json();
+  return d.data?.[0]?.embedding;
+}
+
+async function main() {
+  console.log("\n[BoneCode RAG Integration Tests]\n");
+
+  // ── 1. Embedding API ────────────────────────────────────────────────────────
+  const emb = await embed("function hello() { return 'world'; }");
+  check("Embedding API", emb?.length === 768, `dims=${emb?.length}`);
+
+  // ── 2. Create project + KB ──────────────────────────────────────────────────
+  let r = await fetch(`${BASE}/projects`, {
+    method: "POST", headers: auth,
+    body: JSON.stringify({ root_commit_hash: "rag-test-" + Date.now(), worktree: "C:/tmp/rag-test", vcs: "git", name: "rag-test", sandboxes: [] }),
+  });
+  const proj = await r.json();
+  check("Create project", r.status === 201, proj.id);
+  const projectId = proj.id;
+
+  r = await fetch(`${BASE}/knowledge-bases`, {
+    method: "POST", headers: auth,
+    body: JSON.stringify({
+      project_id: projectId, name: "rag-test-kb",
+      embedding_model: EMBED_MODEL, chunk_strategy: "ast",
+      embedding_dimensions: 768, chunk_size: 1500, chunk_overlap: 150,
+      total_files: 0, total_chunks: 0,
+    }),
+  });
+  const kb = await r.json();
+  check("Create knowledge base", r.status === 201, kb.id);
+  const kbId = kb.id;
+
+  // ── 3. Create code file ─────────────────────────────────────────────────────
+  r = await fetch(`${BASE}/code-files`, {
+    method: "POST", headers: auth,
+    body: JSON.stringify({
+      knowledge_base_id: kbId, project_id: projectId,
+      file_path: "C:/tmp/rag-test/hello.ts", language: "typescript",
+      file_hash: "a".repeat(64), file_size_bytes: 200,
+      total_chunks: 2, last_modified_at: new Date().toISOString(),
+    }),
+  });
+  const cf = await r.json();
+  check("Create code file", r.status === 201, cf.id);
+  const codeFileId = cf.id;
+
+  // ── 4. Insert chunks with real embeddings ───────────────────────────────────
+  const chunks = [
+    { content: "function hello() { return 'world'; }", symbol_name: "hello", symbol_type: "function" },
+    { content: "class Greeter { greet(name: string) { return `Hello ${name}`; } }", symbol_name: "Greeter", symbol_type: "class" },
+  ];
+
+  for (let i = 0; i < chunks.length; i++) {
+    const c = chunks[i];
+    const embedding = await embed(c.content);
+    const embStr = "[" + embedding.join(",") + "]";
+    await pool.query(
+      `INSERT INTO code_chunks (id, code_file_id, knowledge_base_id, project_id, content, start_line, end_line, chunk_index, language, symbol_name, symbol_type, embedding)
+       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12::vector)`,
+      [uuid(), codeFileId, kbId, projectId, c.content, i * 10 + 1, i * 10 + 5, i, "typescript", c.symbol_name, c.symbol_type, embStr]
+    );
+    check(`Insert chunk ${i + 1} with embedding`, true, c.symbol_name);
+  }
+
+  // ── 5. Hybrid search via stored function ────────────────────────────────────
+  const queryEmb = await embed("how to greet someone");
+  const queryEmbStr = "[" + queryEmb.join(",") + "]";
+
+  const searchResult = await pool.query(
+    "SELECT * FROM bonecode_hybrid_search($1, $2::vector, $3, NULL, NULL, 5)",
+    [kbId, queryEmbStr, "greet hello"]
+  );
+  check("Hybrid search returns results", searchResult.rows.length > 0, `count=${searchResult.rows.length}`);
+
+  if (searchResult.rows.length > 0) {
+    const top = searchResult.rows[0];
+    check("Top result has content", !!top.content, top.content?.slice(0, 50));
+    check("Top result has rrf_score > 0", top.rrf_score > 0, `score=${parseFloat(top.rrf_score).toFixed(4)}`);
+    check("Top result has vector_score", top.vector_score > 0, `vscore=${parseFloat(top.vector_score).toFixed(4)}`);
+  }
+
+  // ── 6. Memory entry with embedding ─────────────────────────────────────────
+  const sessionId = uuid(); // fake session for memory test
+  const memEmb = await embed("The user prefers TypeScript over JavaScript");
+  const memEmbStr = "[" + memEmb.join(",") + "]";
+
+  await pool.query(
+    `INSERT INTO memory_entrys (id, session_id, project_id, memory_type, content, importance, access_count, state, embedding)
+     VALUES ($1, $2, $3, $4, $5, $6, 0, 'active', $7::vector)`,
+    [uuid(), sessionId, projectId, "preference", "The user prefers TypeScript over JavaScript", 0.8, memEmbStr]
+  );
+  check("Insert memory entry with embedding", true);
+
+  const memSearch = await pool.query(
+    "SELECT * FROM bonecode_memory_search($1, $2, $3::vector, $4, NULL, 5)",
+    [sessionId, projectId, memEmbStr, "TypeScript preference"]
+  );
+  check("Memory search returns results", memSearch.rows.length > 0, `count=${memSearch.rows.length}`);
+
+  // ── 7. LLM chat with RAG context ────────────────────────────────────────────
+  const ragContext = searchResult.rows.map((row, i) =>
+    `[${i + 1}] ${row.file_path}:${row.start_line}-${row.end_line}\n\`\`\`typescript\n${row.content}\n\`\`\``
+  ).join("\n\n");
+
+  const systemPrompt = `You are a code assistant. Here is relevant code context:\n\n${ragContext}`;
+  const llmR = await fetch(`${LLM_BASE}/chat/completions`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json", Authorization: "Bearer not-needed" },
+    body: JSON.stringify({
+      model: CHAT_MODEL,
+      messages: [
+        { role: "system", content: systemPrompt },
+        { role: "user", content: "What does the hello function return?" },
+      ],
+      max_tokens: 512,
+      temperature: 0,
+    }),
+  });
+  const llmData = await llmR.json();
+  const msg = llmData.choices?.[0]?.message;
+  // Qwen3 reasoning models put final answer in content, thinking in reasoning_content
+  const reply = (msg?.content?.trim() || msg?.reasoning_content?.trim() || "").slice(0, 200);
+  check("LLM answers with RAG context", llmR.status === 200 && !!reply, `reply="${reply?.slice(0, 60)}"`);
+
+  // ── 8. Full /v2/session prompt flow ─────────────────────────────────────────
+  // Create a real session and send a prompt through the compat API
+  r = await fetch(`${BASE}/v2/session`, {
+    method: "POST", headers: auth,
+    body: JSON.stringify({ title: "Prompt Test", directory: "C:/tmp" }),
+  });
+  const sess = await r.json();
+  check("Create session for prompt test", r.status === 201, sess.id);
+
+  // Note: /v2/session/:id/prompt streams SSE — we just verify it starts correctly
+  const promptController = new AbortController();
+  const promptTimeout = setTimeout(() => promptController.abort(), 8000);
+  try {
+    r = await fetch(`${BASE}/v2/session/${sess.id}/prompt`, {
+      method: "POST", headers: auth,
+      body: JSON.stringify({ content: "What is 2+2?", modelID: CHAT_MODEL, providerID: "openai_compatible" }),
+      signal: promptController.signal,
+    });
+    clearTimeout(promptTimeout);
+    check("POST /v2/session/:id/prompt starts", r.status === 200, `content-type=${r.headers.get("content-type")}`);
+  } catch (e) {
+    clearTimeout(promptTimeout);
+    // AbortError means SSE started streaming (expected)
+    check("POST /v2/session/:id/prompt starts (SSE)", e.name === "AbortError", e.name);
+  }
+
+  // ── Summary ─────────────────────────────────────────────────────────────────
+  await pool.end();
+  console.log("\n-------------------------------------");
+  console.log(`Results: ${pass} passed, ${fail} failed`);
+  if (fail > 0) process.exit(1);
+}
+
+main().catch(e => { console.error("FATAL:", e.message, e.stack); process.exit(1); });

package/scripts/test_stream_events.js

@@ -0,0 +1,36 @@
+require("dotenv").config();
+const { streamText } = require("ai");
+const { createOpenAI } = require("@ai-sdk/openai");
+const { z } = require("zod");
+
+(async () => {
+  const model = createOpenAI({
+    apiKey: process.env.OPENAI_API_KEY || "not-needed",
+    baseURL: process.env.OPENAI_BASE_URL,
+  })(process.env.DEFAULT_MODEL);
+
+  console.log("Streaming from:", process.env.DEFAULT_MODEL);
+
+  const result = streamText({
+    model,
+    messages: [{ role: "user", content: "Say hello in one word." }],
+    maxTokens: 200,
+    temperature: 0,
+  });
+
+  const events = [];
+  for await (const event of result.fullStream) {
+    events.push({ type: event.type, data: JSON.stringify(event).slice(0, 100) });
+    console.log("Event:", event.type, JSON.stringify(event).slice(0, 120));
+  }
+
+  console.log("\nTotal events:", events.length);
+  console.log("Event types:", [...new Set(events.map(e => e.type))].join(", "));
+
+  try {
+    const text = await result.text;
+    console.log("result.text:", JSON.stringify(text));
+  } catch (e) {
+    console.log("result.text error:", e.message);
+  }
+})().catch(e => console.error("FATAL:", e.message));

package/scripts/test_websocket_and_saga.js

@@ -0,0 +1,216 @@
+/**
+ * BoneCode: WebSocket + Ingestion Saga Integration Test
+ *
+ * Tests:
+ *   1. WebSocket connection to session_events channel
+ *   2. Receiving events via WebSocket when session state changes
+ *   3. IndexingJob creation → worker picks it up → saga runs → KB transitions to ready
+ *   4. State machine enforcement (invalid transitions rejected)
+ */
+require("dotenv").config();
+const { Pool } = require("pg");
+const { v4: uuid } = require("uuid");
+const WebSocket = require("ws");
+const jwt = require("jsonwebtoken");
+const fs = require("fs");
+const path = require("path");
+const os = require("os");
+
+const pool = new Pool({ connectionString: process.env.DATABASE_URL });
+const BASE = "http://localhost:3000";
+const WS_BASE = "ws://localhost:3000";
+const token = jwt.sign({ sub: "test-user" }, process.env.JWT_SECRET, { expiresIn: "1h" });
+const auth = { Authorization: "Bearer " + token, "Content-Type": "application/json" };
+
+let pass = 0, fail = 0;
+const check = (label, ok, detail = "") => {
+  console.log((ok ? "PASS" : "FAIL"), label, detail || "");
+  ok ? pass++ : fail++;
+};
+
+async function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
+
+async function main() {
+  console.log("\n[BoneCode WebSocket + Saga Tests]\n");
+
+  // ── 1. WebSocket: connect to session_events channel ─────────────────────────
+  const wsEvents = [];
+  const ws = new WebSocket(`${WS_BASE}/ws?channel=session_events&token=${token}`);
+
+  await new Promise((resolve, reject) => {
+    ws.on("open", resolve);
+    ws.on("error", reject);
+    setTimeout(() => reject(new Error("WS connect timeout")), 5000);
+  });
+  check("WebSocket connects to session_events", ws.readyState === WebSocket.OPEN);
+
+  ws.on("message", (data) => {
+    try { wsEvents.push(JSON.parse(data.toString())); } catch {}
+  });
+
+  // Wait for the 'connected' message
+  await sleep(500);
+  const connMsg = wsEvents.find(e => e.type === "connected");
+  check("WebSocket receives 'connected' message", !!connMsg, connMsg ? `channel=${connMsg.channel}` : "");
+
+  // ── 2. WebSocket: create session and verify event arrives ───────────────────
+  const r1 = await fetch(`${BASE}/v2/session`, {
+    method: "POST", headers: auth,
+    body: JSON.stringify({ title: "WS Test Session", directory: "C:/tmp" }),
+  });
+  const sess = await r1.json();
+  check("Create session for WS test", r1.status === 201, sess.id);
+
+  // Give the event bus time to deliver
+  await sleep(1000);
+
+  // The SessionCreated event should have been broadcast to the WS channel
+  const sessionEvent = wsEvents.find(e => e.session_id === sess.id || e.id === sess.id);
+  check("SessionCreated event received via WebSocket", !!sessionEvent, sessionEvent ? JSON.stringify(sessionEvent).slice(0, 80) : "no event");
+
+  // ── 3. State machine: invalid transition rejected ───────────────────────────
+  // Sessions start in 'created' state. Trying to go directly to 'archived' should fail.
+  const r2 = await fetch(`${BASE}/sessions/${sess.id}`, {
+    method: "PUT", headers: auth,
+    body: JSON.stringify({ state: "archived" }),
+  });
+  // created → archived is not a valid transition (must go created → active → ... → archived)
+  check("Invalid state transition rejected (created→archived)", r2.status === 422, `status=${r2.status}`);
+
+  // Valid transition: created → active
+  const r3 = await fetch(`${BASE}/sessions/${sess.id}`, {
+    method: "PUT", headers: auth,
+    body: JSON.stringify({ state: "active" }),
+  });
+  check("Valid state transition accepted (created→active)", r3.status === 200, `status=${r3.status}`);
+
+  // ── 4. Ingestion saga: create KB + job, worker processes it ─────────────────
+  // Create a temp directory with some test files
+  const tmpDir = path.join(os.tmpdir(), "bonecode-test-" + Date.now());
+  fs.mkdirSync(tmpDir, { recursive: true });
+  fs.writeFileSync(path.join(tmpDir, "hello.ts"), `
+export function hello(name: string): string {
+  return \`Hello, \${name}!\`;
+}
+
+export class Greeter {
+  constructor(private prefix: string) {}
+  greet(name: string): string {
+    return \`\${this.prefix} \${name}\`;
+  }
+}
+`);
+  fs.writeFileSync(path.join(tmpDir, "utils.ts"), `
+export function add(a: number, b: number): number {
+  return a + b;
+}
+
+export function multiply(a: number, b: number): number {
+  return a * b;
+}
+`);
+
+  // Create project + KB
+  const projR = await fetch(`${BASE}/projects`, {
+    method: "POST", headers: auth,
+    body: JSON.stringify({ root_commit_hash: "saga-test-" + Date.now(), worktree: tmpDir, vcs: "git", name: "saga-test", sandboxes: [] }),
+  });
+  const proj = await projR.json();
+  check("Create project for saga test", projR.status === 201, proj.id);
+
+  const kbR = await fetch(`${BASE}/knowledge-bases`, {
+    method: "POST", headers: auth,
+    body: JSON.stringify({
+      project_id: proj.id, name: "saga-test-kb",
+      embedding_model: process.env.EMBEDDING_MODEL || "text-embedding-nomic-embed-text-v1.5",
+      chunk_strategy: "ast", embedding_dimensions: 768,
+      chunk_size: 1500, chunk_overlap: 150, total_files: 0, total_chunks: 0,
+    }),
+  });
+  const kb = await kbR.json();
+  check("Create knowledge base for saga test", kbR.status === 201, kb.id);
+  check("KB starts in 'empty' state", kb.state === "empty", `state=${kb.state}`);
+
+  // Create an indexing job — this is what the worker picks up
+  const jobR = await fetch(`${BASE}/indexing-jobs`, {
+    method: "POST", headers: auth,
+    body: JSON.stringify({
+      knowledge_base_id: kb.id, project_id: proj.id,
+      job_type: "full_index",
+      file_paths: JSON.stringify([path.join(tmpDir, "hello.ts"), path.join(tmpDir, "utils.ts")]),
+      files_total: 2, files_processed: 0, chunks_total: 0, chunks_embedded: 0,
+    }),
+  });
+  const job = await jobR.json();
+  check("Create indexing job", jobR.status === 201, `id=${job.id} state=${job.state}`);
+  check("Job starts in 'queued' state", job.state === "queued", `state=${job.state}`);
+
+  // Wait for the worker to process it (worker polls every 2s)
+  console.log("  Waiting for worker to process job (up to 30s)...");
+  let kbReady = false;
+  for (let i = 0; i < 15; i++) {
+    await sleep(2000);
+    const kbCheck = await fetch(`${BASE}/knowledge-bases/${kb.id}`, { headers: auth });
+    const kbData = await kbCheck.json();
+    if (kbData.state === "ready") { kbReady = true; break; }
+    if (kbData.state === "error") { check("KB saga failed", false, `state=${kbData.state}`); break; }
+    process.stdout.write(`  KB state: ${kbData.state}...\r`);
+  }
+  console.log("");
+
+  check("KB transitions to 'ready' after saga", kbReady, kbReady ? "state=ready" : "timed out");
+
+  if (kbReady) {
+    // Verify chunks were created
+    const chunksR = await fetch(`${BASE}/knowledge-bases/${kb.id}/files`, { headers: auth });
+    const chunksData = await chunksR.json();
+    check("Code files indexed", chunksData.total >= 2, `files=${chunksData.total}`);
+
+    const chunkCount = await pool.query(
+      "SELECT COUNT(*) as count FROM code_chunks WHERE knowledge_base_id = $1",
+      [kb.id]
+    );
+    const numChunks = parseInt(chunkCount.rows[0].count);
+    check("Code chunks created with embeddings", numChunks > 0, `chunks=${numChunks}`);
+
+    const embeddedCount = await pool.query(
+      "SELECT COUNT(*) as count FROM code_chunks WHERE knowledge_base_id = $1 AND embedding IS NOT NULL",
+      [kb.id]
+    );
+    const numEmbedded = parseInt(embeddedCount.rows[0].count);
+    check("All chunks have embeddings", numEmbedded === numChunks, `embedded=${numEmbedded}/${numChunks}`);
+
+    // Verify hybrid search works on the indexed content
+    const embR = await fetch("http://10.0.0.20:1234/v1/embeddings", {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: "Bearer not-needed" },
+      body: JSON.stringify({ model: process.env.EMBEDDING_MODEL, input: "greet someone by name" }),
+    });
+    const embData = await embR.json();
+    const qEmb = embData.data?.[0]?.embedding;
+    const searchResult = await pool.query(
+      "SELECT * FROM bonecode_hybrid_search($1, $2::vector, $3, NULL, NULL, 5)",
+      [kb.id, "[" + qEmb.join(",") + "]", "greet name"]
+    );
+    check("Hybrid search finds indexed content", searchResult.rows.length > 0, `results=${searchResult.rows.length}`);
+    if (searchResult.rows.length > 0) {
+      check("Top result is relevant", searchResult.rows[0].content.includes("greet") || searchResult.rows[0].content.includes("Greeter"), searchResult.rows[0].content.slice(0, 60));
+    }
+  }
+
+  // ── 5. Verify job state machine ─────────────────────────────────────────────
+  const jobCheck = await fetch(`${BASE}/indexing-jobs/${job.id}`, { headers: auth });
+  const jobData = await jobCheck.json();
+  check("Job transitions to 'done' after saga", jobData.state === "done", `state=${jobData.state}`);
+
+  // ── Cleanup ──────────────────────────────────────────────────────────────────
+  ws.close();
+  fs.rmSync(tmpDir, { recursive: true, force: true });
+  await pool.end();
+
+  console.log("\n-------------------------------------");
+  console.log(`Results: ${pass} passed, ${fail} failed`);
+  if (fail > 0) process.exit(1);
+}
+
+main().catch(e => { console.error("FATAL:", e.message, e.stack); process.exit(1); });