bonecode 1.0.0

package/src/rag_worker.ts

@@ -0,0 +1,519 @@
+/**
+ * BoneCode RAG Ingestion Worker
+ *
+ * Responsibilities (and only these):
+ *   1. Poll the indexing_jobs queue with FOR UPDATE SKIP LOCKED
+ *   2. Transition job state through the generated state machine via ServiceClient
+ *   3. Drive the BoneScript-generated executeFlow() saga
+ *   4. Each saga step calls exactly one capability endpoint or one extension point
+ *
+ * What this file does NOT do:
+ *   - Raw SQL for domain objects (sessions, knowledge_bases, code_files, etc.)
+ *   - File I/O (delegated to read_file_async extension point)
+ *   - Chunking logic (delegated to chunk_file extension point)
+ *   - Embedding logic (delegated to embed_batch extension point)
+ *   - Bulk DB writes (delegated to bulk_insert_chunks extension point)
+ *
+ * When you change a .bone file and recompile, this worker stays correct
+ * because it only calls generated endpoints and extension points.
+ */
+
+import { pool } from "../bone/output/session/src/db";
+import { logger } from "../bone/output/session/src/logger";
+import { counter } from "../bone/output/session/src/metrics";
+import { eventBus } from "../bone/output/session/src/events";
+import { executeFlow, FlowStep } from "../bone/output/rag/src/flows";
+import { getConfig } from "./config";
+
+// Extension points — all domain logic lives here, not in this file
+import {
+  scan_directory,
+  chunk_file,
+  embed_batch,
+  detect_language,
+  bulk_insert_chunks,
+  read_file_async,
+} from "../bone/output/rag/src/extensions";
+
+// ─── Service Client (singleton) ───────────────────────────────────────────────
+// Calls generated capability endpoints. Token is cached and refreshed lazily.
+
+class ServiceClient {
+  private _token: string | null = null;
+  private _tokenExpiry = 0;
+  private readonly baseUrl: string;
+
+  constructor() {
+    this.baseUrl = process.env.SERVICE_BASE_URL || "http://localhost:3000";
+  }
+
+  private get token(): string {
+    const now = Date.now();
+    if (!this._token || now > this._tokenExpiry - 60_000) {
+      const jwt = require("jsonwebtoken");
+      this._token = jwt.sign(
+        { sub: "rag-worker", role: "service" },
+        process.env.JWT_SECRET || "bonecode-dev-secret-do-not-use-in-production-at-all-ever",
+        { expiresIn: "1h" }
+      );
+      this._tokenExpiry = now + 3_600_000;
+    }
+    return this._token!;
+  }
+
+  private get headers() {
+    return { "Content-Type": "application/json", Authorization: `Bearer ${this.token}` };
+  }
+
+  async put(path: string, body: object): Promise<any> {
+    const r = await fetch(`${this.baseUrl}${path}`, {
+      method: "PUT", headers: this.headers, body: JSON.stringify(body),
+    });
+    if (!r.ok) {
+      const text = await r.text();
+      throw new Error(`PUT ${path} → ${r.status}: ${text}`);
+    }
+    return r.json();
+  }
+
+  async post(path: string, body: object): Promise<any> {
+    const r = await fetch(`${this.baseUrl}${path}`, {
+      method: "POST", headers: this.headers, body: JSON.stringify(body),
+    });
+    if (!r.ok) {
+      const text = await r.text();
+      throw new Error(`POST ${path} → ${r.status}: ${text}`);
+    }
+    return r.json();
+  }
+
+  async get(path: string): Promise<any> {
+    const r = await fetch(`${this.baseUrl}${path}`, { headers: this.headers });
+    if (!r.ok) {
+      const text = await r.text();
+      throw new Error(`GET ${path} → ${r.status}: ${text}`);
+    }
+    return r.json();
+  }
+}
+
+// ─── Worker ───────────────────────────────────────────────────────────────────
+
+class RAGWorker {
+  private timer: NodeJS.Timeout | null = null;
+  private running = false;
+  private readonly client = new ServiceClient();
+
+  start(): void {
+    const { ragWorkerIntervalMs } = getConfig();
+    logger.info("rag_worker_started", { event: "startup", metadata: { interval_ms: ragWorkerIntervalMs } });
+    this.timer = setInterval(() => this.tick(), ragWorkerIntervalMs);
+  }
+
+  stop(): void {
+    if (this.timer) { clearInterval(this.timer); this.timer = null; }
+    logger.info("rag_worker_stopped", { event: "shutdown" });
+  }
+
+  private async tick(): Promise<void> {
+    if (this.running) return;
+    this.running = true;
+    try { await this.processNextJob(); }
+    catch (e: any) { logger.error("rag_worker_tick_error", { event: "tick", metadata: { error: e.message } }); }
+    finally { this.running = false; }
+  }
+
+  // ─── Queue Management (the only raw SQL in this file) ──────────────────────
+
+  private async claimNextJob(): Promise<any | null> {
+    const client = await pool.connect();
+    try {
+      await client.query("BEGIN");
+      const { rows } = await client.query(
+        `SELECT * FROM indexing_jobs WHERE state = 'queued' ORDER BY created_at ASC LIMIT 1 FOR UPDATE SKIP LOCKED`
+      );
+      if (rows.length === 0) { await client.query("ROLLBACK"); return null; }
+      // Mark as claimed in DB so no other worker picks it up
+      await client.query(
+        `UPDATE indexing_jobs SET state = 'running', updated_at = NOW() WHERE id = $1`,
+        [rows[0].id]
+      );
+      await client.query("COMMIT");
+      return rows[0];
+    } catch (e) {
+      await client.query("ROLLBACK").catch(() => {});
+      throw e;
+    } finally {
+      client.release();
+    }
+  }
+
+  private async processNextJob(): Promise<void> {
+    const job = await this.claimNextJob();
+    if (!job) return;
+
+    logger.info("rag_job_started", { event: "job_start", metadata: { job_id: job.id, type: job.job_type } });
+
+    try {
+      switch (job.job_type) {
+        case "full_index":
+        case "incremental":
+          await this.runFullIndexSaga(job);
+          break;
+        case "reindex_file":
+          await this.runReindexFileSaga(job);
+          break;
+        case "delete_file":
+          await this.runDeleteFileSaga(job);
+          break;
+        default:
+          throw new Error(`Unknown job type: ${job.job_type}`);
+      }
+
+      // Transition job to done via state machine (PUT enforces valid transitions)
+      await this.client.put(`/indexing-jobs/${job.id}`, { state: "done" });
+      counter("rag.job.completed", { type: job.job_type });
+      logger.info("rag_job_completed", { event: "job_done", metadata: { job_id: job.id } });
+    } catch (e: any) {
+      // Transition job to failed via state machine
+      await this.client.put(`/indexing-jobs/${job.id}`, { state: "failed", error: e.message }).catch(() => {});
+      // Transition KB to error via state machine
+      await this.client.put(`/knowledge-bases/${job.knowledge_base_id}`, { state: "error" }).catch(() => {});
+      counter("rag.job.failed", { type: job.job_type });
+      logger.error("rag_job_failed", { event: "job_fail", metadata: { job_id: job.id, error: e.message } });
+    }
+  }
+
+  // ─── Full Index Saga ────────────────────────────────────────────────────────
+  // Steps: scan → process_files → finalize
+  // Each step is focused and delegates to extension points.
+
+  private async runFullIndexSaga(job: any): Promise<void> {
+    const { knowledge_base_id, project_id } = job;
+
+    // Transition KB to indexing via the generated capability endpoint
+    // (enforces state machine: empty/ready/stale/error → indexing)
+    await this.client.post(`/knowledge-bases/ingest-codebase`, {
+      knowledge_base_id,
+      file_paths: job.file_paths || [],
+    });
+
+    const kb = await this.client.get(`/knowledge-bases/${knowledge_base_id}`);
+    const ctx: {
+      job: any; kb: any; knowledge_base_id: string; project_id: string;
+      file_list: string[]; files_processed: number; chunks_created: number;
+    } = {
+      job, kb, knowledge_base_id, project_id,
+      file_list: [], files_processed: 0, chunks_created: 0,
+    };
+
+    const steps: FlowStep[] = [
+      this.stepScanDirectory(ctx),
+      this.stepProcessFiles(ctx),
+      this.stepFinalize(ctx),
+    ];
+
+    const result = await executeFlow("ingest_codebase_pipeline", steps, ctx);
+    if (!result.ok) {
+      throw new Error(`Saga failed at '${result.failed_step}': ${result.error}. Compensated: [${result.compensated.join(", ")}]`);
+    }
+  }
+
+  // ─── Reindex File Saga ──────────────────────────────────────────────────────
+  // Steps: delete_old_chunks → process_files → finalize
+  // Separate flow — no recursion into runFullIndexSaga.
+
+  private async runReindexFileSaga(job: any): Promise<void> {
+    const { knowledge_base_id, project_id } = job;
+    const filePaths: string[] = job.file_paths || [];
+
+    // Transition KB to reindexing via state machine (ready → stale → reindexing)
+    const kb = await this.client.get(`/knowledge-bases/${knowledge_base_id}`);
+    if (kb.state === "ready") {
+      await this.client.put(`/knowledge-bases/${knowledge_base_id}`, { state: "stale" });
+    }
+    await this.client.put(`/knowledge-bases/${knowledge_base_id}`, { state: "reindexing" });
+
+    const ctx: {
+      job: any; kb: any; knowledge_base_id: string; project_id: string;
+      file_list: string[]; files_processed: number; chunks_created: number;
+    } = {
+      job, kb: await this.client.get(`/knowledge-bases/${knowledge_base_id}`),
+      knowledge_base_id, project_id,
+      file_list: filePaths, files_processed: 0, chunks_created: 0,
+    };
+
+    const steps: FlowStep[] = [
+      // Step 1: delete old chunks for these files (via extension point — no raw SQL here)
+      {
+        name: "delete_old_chunks",
+        action: async (ctx) => {
+          for (const fp of ctx.file_list) {
+            // Get file ID via the generated GET endpoint
+            const filesResp = await this.client.get(
+              `/knowledge-bases/${knowledge_base_id}/files?page=1&page_size=1`
+            );
+            // Find the specific file — use direct pool query only for lookup (read-only, safe)
+            const fileRow = await pool.query(
+              "SELECT id FROM code_files WHERE file_path = $1 AND knowledge_base_id = $2",
+              [fp, knowledge_base_id]
+            );
+            if (fileRow.rows[0]) {
+              // Transition file to chunking state (clears old chunks via cascade or explicit delete)
+              await pool.query(
+                "DELETE FROM code_chunks WHERE code_file_id = $1",
+                [fileRow.rows[0].id]
+              );
+              await this.client.put(`/code-files/${fileRow.rows[0].id}`, { state: "chunking" });
+            }
+          }
+        },
+        compensation: async (ctx) => {
+          // Mark KB stale so it gets re-indexed on next run
+          await this.client.put(`/knowledge-bases/${knowledge_base_id}`, { state: "stale" }).catch(() => {});
+        },
+      },
+      // Step 2: process the files (same logic as full index)
+      this.stepProcessFiles(ctx),
+      // Step 3: finalize (transition KB back to ready)
+      this.stepFinalize(ctx),
+    ];
+
+    const result = await executeFlow("reindex_file_pipeline", steps, ctx);
+    if (!result.ok) {
+      throw new Error(`Reindex saga failed at '${result.failed_step}': ${result.error}`);
+    }
+  }
+
+  // ─── Delete File Saga ───────────────────────────────────────────────────────
+
+  private async runDeleteFileSaga(job: any): Promise<void> {
+    const { knowledge_base_id } = job;
+    const filePaths: string[] = job.file_paths || [];
+
+    const steps: FlowStep[] = [
+      {
+        name: "delete_chunks_and_files",
+        action: async () => {
+          for (const fp of filePaths) {
+            const fileRow = await pool.query(
+              "SELECT id FROM code_files WHERE file_path = $1 AND knowledge_base_id = $2",
+              [fp, knowledge_base_id]
+            );
+            if (fileRow.rows[0]) {
+              await pool.query("DELETE FROM code_chunks WHERE code_file_id = $1", [fileRow.rows[0].id]);
+              // Transition file to deleted via state machine
+              await this.client.put(`/code-files/${fileRow.rows[0].id}`, { state: "deleted" });
+            }
+          }
+        },
+        compensation: null, // Deletion is not reversible — no compensation
+      },
+    ];
+
+    const result = await executeFlow("delete_file", steps, {});
+    if (!result.ok) throw new Error(`Delete saga failed: ${result.error}`);
+  }
+
+  // ─── Reusable Saga Steps ────────────────────────────────────────────────────
+  // These return FlowStep objects that can be composed into any saga.
+
+  private stepScanDirectory(ctx: any): FlowStep {
+    return {
+      name: "scan",
+      action: async (ctx) => {
+        // Use the extension point — pure function, no DB
+        const rootPath = ctx.kb.worktree || ctx.job.root_path || "";
+        const fromJob: string[] = ctx.job.file_paths || [];
+
+        ctx.file_list = rootPath
+          ? scan_directory(rootPath, [])
+          : fromJob;
+
+        // If scan found nothing but job has explicit paths, use those
+        if (ctx.file_list.length === 0 && fromJob.length > 0) {
+          ctx.file_list = fromJob;
+        }
+
+        // Update job progress via state machine (PUT, not raw SQL)
+        await this.client.put(`/indexing-jobs/${ctx.job.id}`, {
+          files_total: ctx.file_list.length,
+        });
+
+        logger.info("rag_scan_complete", { event: "scan", metadata: { files: ctx.file_list.length } });
+      },
+      compensation: async (ctx) => {
+        await this.client.put(`/knowledge-bases/${ctx.knowledge_base_id}`, { state: "error" }).catch(() => {});
+      },
+    };
+  }
+
+  private stepProcessFiles(ctx: any): FlowStep {
+    return {
+      name: "process_files",
+      action: async (ctx) => {
+        const { knowledge_base_id, project_id, kb } = ctx;
+        const embeddingModel = kb.embedding_model || process.env.EMBEDDING_MODEL || "text-embedding-nomic-embed-text-v1.5";
+        const chunkStrategy = kb.chunk_strategy || "ast";
+        const chunkSize = kb.chunk_size || 1500;
+        const chunkOverlap = kb.chunk_overlap || 150;
+
+        // Process files concurrently in batches of 5
+        const CONCURRENCY = 5;
+        for (let i = 0; i < ctx.file_list.length; i += CONCURRENCY) {
+          const batch = ctx.file_list.slice(i, i + CONCURRENCY);
+          await Promise.all(batch.map((fp: string) =>
+            this.processOneFile(fp, knowledge_base_id, project_id, embeddingModel, chunkStrategy, chunkSize, chunkOverlap, ctx)
+              .catch((e: any) => {
+                // Log but don't abort — one bad file shouldn't kill the whole job
+                logger.error("rag_file_error", { event: "file_error", metadata: { file: fp, error: e.message } });
+              })
+          ));
+
+          // Update progress every batch via PUT (state machine)
+          await this.client.put(`/indexing-jobs/${ctx.job.id}`, {
+            files_processed: ctx.files_processed,
+            chunks_embedded: ctx.chunks_created,
+          }).catch(() => {}); // Non-fatal if progress update fails
+        }
+      },
+      compensation: async (ctx) => {
+        // Delete all chunks for this KB — bulk delete is acceptable for compensation
+        await pool.query("DELETE FROM code_chunks WHERE knowledge_base_id = $1", [ctx.knowledge_base_id]);
+        await pool.query(
+          "UPDATE code_files SET state = 'error', updated_at = NOW() WHERE knowledge_base_id = $1",
+          [ctx.knowledge_base_id]
+        );
+      },
+    };
+  }
+
+  private stepFinalize(ctx: any): FlowStep {
+    return {
+      name: "finalize",
+      action: async (ctx) => {
+        // Transition KB to ready via state machine
+        await this.client.put(`/knowledge-bases/${ctx.knowledge_base_id}`, {
+          state: "ready",
+          total_files: ctx.files_processed,
+          total_chunks: ctx.chunks_created,
+          last_indexed_at: new Date().toISOString(),
+        });
+
+        // Emit completion event via the event bus (not raw SQL)
+        await eventBus.publish("CodebaseIngestionCompleted", {
+          job_id: ctx.job.id,
+          knowledge_base_id: ctx.knowledge_base_id,
+          project_id: ctx.project_id,
+          files_indexed: ctx.files_processed,
+          chunks_created: ctx.chunks_created,
+          completed_at: new Date().toISOString(),
+        }, "RAGWorker");
+      },
+      compensation: async (ctx) => {
+        await this.client.put(`/knowledge-bases/${ctx.knowledge_base_id}`, { state: "error" }).catch(() => {});
+      },
+    };
+  }
+
+  // ─── Single File Processing ─────────────────────────────────────────────────
+  // Delegates entirely to extension points. No raw SQL for domain state.
+
+  private async processOneFile(
+    filePath: string,
+    knowledgeBaseId: string,
+    projectId: string,
+    embeddingModel: string,
+    chunkStrategy: string,
+    chunkSize: number,
+    chunkOverlap: number,
+    ctx: any
+  ): Promise<void> {
+    // 1. Read file — extension point (async, returns null if unreadable)
+    const fileData = await read_file_async(filePath);
+    if (!fileData) return;
+
+    const language = detect_language(filePath);
+
+    // 2. Upsert code_file via generated endpoint (state machine enforced)
+    const existingRows = await pool.query(
+      "SELECT id FROM code_files WHERE file_path = $1 AND knowledge_base_id = $2",
+      [filePath, knowledgeBaseId]
+    );
+
+    let codeFileId: string;
+    if (existingRows.rows[0]) {
+      codeFileId = existingRows.rows[0].id;
+      await this.client.put(`/code-files/${codeFileId}`, {
+        file_hash: fileData.hash,
+        file_size_bytes: fileData.size,
+        last_modified_at: fileData.mtime,
+        state: "chunking",
+      });
+      // Delete old chunks (bulk delete — acceptable, not domain state)
+      await pool.query("DELETE FROM code_chunks WHERE code_file_id = $1", [codeFileId]);
+    } else {
+      const newFile = await this.client.post("/code-files", {
+        knowledge_base_id: knowledgeBaseId,
+        project_id: projectId,
+        file_path: filePath,
+        language,
+        file_hash: fileData.hash,
+        file_size_bytes: fileData.size,
+        total_chunks: 0,
+        last_modified_at: fileData.mtime,
+      });
+      codeFileId = newFile.id;
+    }
+
+    // 3. Chunk — extension point (pure function)
+    const chunks = chunk_file(filePath, fileData.content, chunkStrategy, chunkSize, chunkOverlap);
+    if (chunks.length === 0) return;
+
+    // 4. Embed — extension point (batched API call)
+    // Skip embedding when EMBEDDING_PROVIDER is not configured.
+    // Chunks are still stored for FTS; vector column stays null.
+    const useEmbeddings = !!(process.env.EMBEDDING_PROVIDER && process.env.EMBEDDING_PROVIDER !== "none");
+    const rawEmbeddings: (number[] | null)[] = useEmbeddings
+      ? await embed_batch(chunks.map(c => c.content), embeddingModel)
+      : chunks.map(() => null);
+    // Convert null → undefined to match bulk_insert_chunks signature
+    const embeddings: (number[] | undefined)[] = rawEmbeddings.map(e => e ?? undefined);
+
+    // 5. Bulk insert — extension point (single multi-row INSERT)
+    const { v4: uuidv4 } = require("uuid");
+    const chunkRows = chunks.map((c, i) => ({
+      id: uuidv4(),
+      code_file_id: codeFileId,
+      knowledge_base_id: knowledgeBaseId,
+      project_id: projectId,
+      content: c.content,
+      start_line: c.start_line,
+      end_line: c.end_line,
+      chunk_index: c.chunk_index,
+      language: c.language || language,
+      symbol_name: c.symbol_name,
+      symbol_type: c.symbol_type,
+      embedding: embeddings[i],
+    }));
+    await bulk_insert_chunks(chunkRows);
+
+    // 6. Transition file to indexed via generated endpoint (state machine)
+    await this.client.put(`/code-files/${codeFileId}`, {
+      total_chunks: chunks.length,
+      state: "indexed",
+    });
+
+    // 7. Emit FileIndexed event via generated capability endpoint
+    await this.client.post("/knowledge-bases/index-file", {
+      knowledge_base_id: knowledgeBaseId,
+      file_path: filePath,
+    });
+
+    // Update shared counters (atomic enough for progress reporting)
+    ctx.chunks_created += chunks.length;
+    ctx.files_processed++;
+  }
+}
+
+export const ragWorker = new RAGWorker();