@comfanion/usethis_search 4.2.0-dev.4 → 4.3.0-dev.1

package/hooks/message-before.ts

@@ -57,13 +57,17 @@ export function createWorkspaceInjectionHandler(state: SessionState) {
     // Don't inject or prune for sub-agents (title generation, etc.)
     if (state.isSubAgent) return
 
-    // ── Prune: replace old search tool outputs with compact summaries ────
-    // Files are already in workspace injection — no need for big search
-    // output in chat history. This runs even when workspace is empty
-    // (handles case where workspace was cleared but old search outputs remain).
+    // ── Prune & Compact: optimize chat history ────────────────────────────
+    // 1. Prune: replace old tool outputs with compact summaries
+    // 2. Compact: remove old tool calls entirely (keep last N turns)
+    // Files are already in workspace injection — no need for big outputs
+    // in chat history. This runs even when workspace is empty
+    // (handles case where workspace was cleared but old outputs remain).
     const wsConfig = workspaceCache.getConfig()
     if (wsConfig.autoPruneSearch !== false) {
       pruneSearchToolOutputs(output.messages)
+      pruneReadToolOutputs(output.messages)
+      compactOldToolCalls(output.messages)
     }
 
     let entries = workspaceCache.getAll()
@@ -225,7 +229,8 @@ function formatFileWithChunks(
 }
 
 /**
- * Format a single chunk with metadata.
+ * Format a single chunk with metadata and line numbers (cat -n style).
+ * This allows the agent to see exact line numbers without needing grep.
  */
 function formatChunk(entry: ReturnType<typeof workspaceCache.getAll>[0]): string {
   let block = ""
@@ -251,11 +256,21 @@ function formatChunk(entry: ReturnType<typeof workspaceCache.getAll>[0]): string
     block += `<!-- ${meta.join(" | ")} -->\n`
   }
 
-  // Chunk content
+  // Chunk content WITH LINE NUMBERS (cat -n style)
+  // This allows agent to reference exact lines without grep
+  const startLine = entry.metadata?.startLine ?? 1
+  const lines = entry.content.split("\n")
   const lang = entry.metadata?.language || ""
+  
   block += `\`\`\`${lang}\n`
-  block += entry.content
-  if (!entry.content.endsWith("\n")) block += "\n"
+  
+  for (let i = 0; i < lines.length; i++) {
+    const lineNum = startLine + i
+    const lineContent = lines[i]
+    // Format: "  123| line content" (5 chars for line number + tab)
+    block += `${lineNum.toString().padStart(5, " ")}| ${lineContent}\n`
+  }
+  
   block += `\`\`\`\n`
 
   return block
@@ -270,7 +285,7 @@ function findLastUserMessage(messages: Message[]): Message | null {
   return null
 }
 
-// ── Search output pruning ────────────────────────────────────────────────────
+// ── Tool output pruning ─────────────────────────────────────────────────────
 
 /**
  * Minimum output length to consider pruning.
@@ -344,3 +359,208 @@ export function pruneSearchToolOutputs(messages: Message[]): void {
       `${attachedCount} attached to workspace. Full content available via workspace context.]`
   }
 }
+
+/**
+ * Replace read() tool outputs in chat history with compact summaries.
+ *
+ * Why: read() returns full file content or large chunks.
+ * After workspace injection (or auto-attach), the content is already in context.
+ * Keeping the read output wastes tokens — replace it with a 1-line summary.
+ *
+ * Only prunes completed read calls with output longer than MIN_PRUNE_LENGTH.
+ * The last read output is kept (the agent may still be referencing it).
+ */
+export function pruneReadToolOutputs(messages: Message[]): void {
+  // Find all read tool parts (completed, with long output)
+  const readParts: { msgIdx: number; partIdx: number; part: MessagePart }[] = []
+
+  for (let i = 0; i < messages.length; i++) {
+    const msg = messages[i]
+    const parts = Array.isArray(msg.parts) ? msg.parts : []
+
+    for (let j = 0; j < parts.length; j++) {
+      const part = parts[j]
+      if (
+        part.type === "tool" &&
+        (part.tool === "read" || part.tool === "Read") &&
+        part.state?.status === "completed" &&
+        typeof part.state?.output === "string" &&
+        part.state.output.length > MIN_PRUNE_LENGTH
+      ) {
+        readParts.push({ msgIdx: i, partIdx: j, part })
+      }
+    }
+  }
+
+  // Keep the last read output (agent may reference it) — prune the rest
+  if (readParts.length <= 1) return
+
+  const toPrune = readParts.slice(0, -1)
+
+  for (const { part } of toPrune) {
+    const output = part.state.output as string
+
+    // Extract file path from output or input
+    const filePath = part.input?.filePath || extractFilePathFromOutput(output)
+
+    // Check if it's a substituted output (already compact)
+    if (output.startsWith("[File ") || output.startsWith("[Lines ") || output.startsWith("✓ Attached chunk")) {
+      // Already substituted — keep as-is
+      continue
+    }
+
+    // Replace with compact summary
+    part.state.output = `[Read "${filePath || "file"}" — content available in workspace context]`
+  }
+}
+
+/**
+ * Extract file path from read() output.
+ * Output usually starts with file path or has markers.
+ */
+function extractFilePathFromOutput(output: string): string | null {
+  // Try to find file path in first line
+  const firstLine = output.split("\n")[0]
+  
+  // Pattern: "## path/to/file.ts" or "path/to/file.ts"
+  const pathMatch = firstLine.match(/##?\s*(.+?\.(ts|js|go|py|md|txt|yaml|json|tsx|jsx|rs|java|kt|swift|c|cpp|h|cs|rb|php))/)
+  if (pathMatch) {
+    return pathMatch[1].trim()
+  }
+  
+  return null
+}
+
+// ── Tool Call Compaction ────────────────────────────────────────────────────
+
+/**
+ * Remove old tool calls (search/read) from chat history.
+ * 
+ * Strategy:
+ * - Keep last N turns (default: 5) — agent may reference recent calls
+ * - Only compact search/read tools (not edit/write/grep/glob)
+ * - Only compact completed calls with pruned outputs
+ * - Remove both call + output parts
+ * - Add compact marker at start showing how many calls removed
+ * 
+ * Why: Tool calls contain full args (200+ tokens). After pruning outputs,
+ * the calls themselves are redundant — chunks already in workspace.
+ * 
+ * Savings: ~220 tokens per compacted call × N calls = 2K-10K tokens
+ */
+const KEEP_LAST_N_TURNS = 5
+const COMPACT_TOOLS = ['search', 'read', 'Read']
+
+interface ToolCallPair {
+  msgIndex: number
+  callPart: MessagePart
+  outputPart?: MessagePart
+  tool: string
+  status: string
+  turnsSinceEnd: number
+}
+
+/**
+ * Compact old tool calls by removing them from chat history.
+ * Keeps last N turns intact.
+ */
+export function compactOldToolCalls(messages: Message[]): void {
+  // Find all tool call pairs
+  const toolPairs = findToolCallPairs(messages)
+  
+  if (toolPairs.length === 0) return
+  
+  // Calculate turns from end for each pair
+  const totalTurns = messages.length
+  
+  // Filter: only old, completed, search/read with pruned outputs
+  const toCompact = toolPairs.filter(pair => {
+    const turnsFromEnd = totalTurns - pair.msgIndex
+    return (
+      turnsFromEnd > KEEP_LAST_N_TURNS &&
+      pair.status === 'completed' &&
+      COMPACT_TOOLS.includes(pair.tool) &&
+      pair.outputPart &&
+      isPrunedOutput(pair.outputPart.state?.output || '')
+    )
+  })
+  
+  if (toCompact.length === 0) return
+  
+  // Remove tool parts from messages
+  const removedIds = new Set<string>()
+  
+  for (const pair of toCompact) {
+    removedIds.add(pair.callPart.id)
+    if (pair.outputPart) {
+      removedIds.add(pair.outputPart.id)
+    }
+  }
+  
+  // Filter out removed parts from messages
+  for (const msg of messages) {
+    if (!msg.parts || !Array.isArray(msg.parts)) continue
+    msg.parts = msg.parts.filter(part => !removedIds.has(part.id))
+  }
+  
+  // Add compact marker to first user message
+  const firstUserMsg = messages.find(m => m?.info?.role === 'user')
+  if (firstUserMsg && firstUserMsg.parts) {
+    const marker = {
+      type: 'text',
+      text: `<!-- ${toCompact.length} tool calls compacted (search/read results in workspace) -->`,
+      id: 'compact-marker-' + Date.now(),
+    }
+    firstUserMsg.parts.unshift(marker)
+  }
+}
+
+/**
+ * Find all tool call + output pairs in messages.
+ */
+function findToolCallPairs(messages: Message[]): ToolCallPair[] {
+  const pairs: ToolCallPair[] = []
+  
+  for (let i = 0; i < messages.length; i++) {
+    const msg = messages[i]
+    if (!msg.parts || !Array.isArray(msg.parts)) continue
+    
+    for (const part of msg.parts) {
+      if (part.type === 'tool' && part.tool) {
+        const status = part.state?.status || 'unknown'
+        
+        // Find matching output part (usually in same message or next)
+        let outputPart: MessagePart | undefined
+        
+        // Check same message first
+        for (const p of msg.parts) {
+          if (p.type === 'tool' && p.tool === part.tool && p.state?.output && p.id !== part.id) {
+            outputPart = p
+            break
+          }
+        }
+        
+        pairs.push({
+          msgIndex: i,
+          callPart: part,
+          outputPart,
+          tool: part.tool,
+          status,
+          turnsSinceEnd: 0, // Will be calculated in compactOldToolCalls
+        })
+      }
+    }
+  }
+  
+  return pairs
+}
+
+/**
+ * Check if output is pruned (compact format).
+ */
+function isPrunedOutput(output: string): boolean {
+  if (!output) return false
+  
+  // Pruned outputs start with [ or ✓
+  return output.startsWith('[') || output.startsWith('✓')
+}

package/hooks/tool-substitution.ts

@@ -21,6 +21,75 @@
 
 import type { SessionState } from "./types.ts"
 import { workspaceCache, WorkspaceCache } from "../cache/manager.ts"
+import { getIndexer, releaseIndexer } from "../vectorizer/index.ts"
+
+// ── Chunk Detection Helpers ─────────────────────────────────────────────────
+
+/**
+ * Find which chunk contains the given line offset.
+ * 
+ * @param filePath Relative file path
+ * @param offset Line number (0-based as used by read())
+ * @param limit Number of lines to read
+ * @returns Chunk metadata if found, null otherwise
+ */
+async function findChunkByOffset(
+  filePath: string,
+  offset: number,
+  limit?: number
+): Promise<{
+  chunk_id: string
+  content: string
+  chunk_index: number
+  start_line: number
+  end_line: number
+  language?: string
+  function_name?: string
+  class_name?: string
+} | null> {
+  try {
+    const projectRoot = process.cwd()
+    const indexer = await getIndexer(projectRoot, "code")
+    
+    try {
+      // Get all chunks for this file
+      const chunks = await indexer.findChunksByPath(filePath)
+      
+      if (chunks.length === 0) return null
+      
+      // Find chunk that contains this offset
+      // offset is 0-based line number from read()
+      const targetLine = offset
+      const endLine = limit ? offset + limit : offset + 100
+      
+      for (const chunk of chunks) {
+        const chunkStart = chunk.start_line ?? 0
+        const chunkEnd = chunk.end_line ?? Number.MAX_SAFE_INTEGER
+        
+        // Check if offset falls within this chunk
+        if (targetLine >= chunkStart && targetLine <= chunkEnd) {
+          return {
+            chunk_id: chunk.chunk_id || `${filePath}:chunk-${chunk.chunk_index ?? 0}`,
+            content: chunk.content,
+            chunk_index: chunk.chunk_index ?? 0,
+            start_line: chunkStart,
+            end_line: chunkEnd,
+            language: chunk.language,
+            function_name: chunk.function_name,
+            class_name: chunk.class_name,
+          }
+        }
+      }
+      
+      return null
+    } finally {
+      releaseIndexer(projectRoot, "code")
+    }
+  } catch (error) {
+    // Index not available or error — return null
+    return null
+  }
+}
 
 /**
  * Create the tool output substitution handler.
@@ -75,7 +144,8 @@ export function createToolSubstitutionHandler(state: SessionState, cache?: Works
     // is valuable for AI navigation. Only read() is substituted.
     switch (input.tool) {
       case "read":
-        substituteReadOutput(output, wsCache)
+      case "Read":
+        await handleReadSubstitution(output, wsCache)
         break
       // case "grep":  // Disabled — AI needs line numbers and match context
       // case "glob":  // Disabled — discovery tool, paths are metadata not content
@@ -84,16 +154,106 @@ export function createToolSubstitutionHandler(state: SessionState, cache?: Works
 }
 
 /**
- * Substitute read() output if file has chunks in workspace.
+ * Handle read() substitution with smart chunk detection.
+ * 
+ * Two modes:
+ * 1. Full read (no offset) → substitute if file in workspace
+ * 2. Partial read (with offset) → auto-attach chunk, then substitute
+ */
+async function handleReadSubstitution(
+  output: { title: string; output: string; metadata: any },
+  cache: WorkspaceCache
+): Promise<void> {
+  const filePath = output.metadata?.filePath || extractFilePathFromTitle(output.title)
+  if (!filePath) return
+  
+  const offset = output.metadata?.offset
+  const limit = output.metadata?.limit
+  const isPartialRead = offset !== undefined
+  
+  // MODE 1: Partial read with offset → auto-attach chunk
+  if (isPartialRead) {
+    await handlePartialReadAttach(filePath, offset, limit, output, cache)
+    return
+  }
+  
+  // MODE 2: Full read → standard substitution
+  substituteReadOutput(output, cache)
+}
+
+/**
+ * Auto-attach chunk for partial read() with offset.
+ * 
+ * When agent does read({ filePath: "src/auth.ts", offset: 150, limit: 50 }),
+ * we find which chunk contains lines 150-200 and attach it to workspace.
+ */
+async function handlePartialReadAttach(
+  filePath: string,
+  offset: number,
+  limit: number | undefined,
+  output: { title: string; output: string; metadata: any },
+  cache: WorkspaceCache
+): Promise<void> {
+  try {
+    // Find which chunk contains this offset
+    const chunk = await findChunkByOffset(filePath, offset, limit)
+    
+    if (!chunk) {
+      // Chunk not found (file not indexed or offset out of range)
+      // Keep original output
+      return
+    }
+    
+    // Check if chunk already in workspace
+    const existing = cache.get(chunk.chunk_id)
+    if (existing) {
+      // Already attached → replace output with reference
+      output.output = `[Lines ${chunk.start_line}-${chunk.end_line} (chunk ${chunk.chunk_index}) already in workspace — see <workspace_context>]`
+      return
+    }
+    
+    // Attach chunk to workspace
+    cache.attach({
+      chunkId: chunk.chunk_id,
+      path: filePath,
+      content: chunk.content,
+      chunkIndex: chunk.chunk_index,
+      role: "manual",
+      attachedAt: Date.now(),
+      attachedBy: `read(offset:${offset})`,
+      metadata: {
+        language: chunk.language,
+        function_name: chunk.function_name,
+        class_name: chunk.class_name,
+        startLine: chunk.start_line,
+        endLine: chunk.end_line,
+      },
+    })
+    
+    // Replace output with compact message
+    const meta: string[] = []
+    if (chunk.function_name) meta.push(`fn: ${chunk.function_name}`)
+    if (chunk.class_name) meta.push(`class: ${chunk.class_name}`)
+    const metaStr = meta.length > 0 ? ` (${meta.join(", ")})` : ""
+    
+    output.output = `✓ Attached chunk ${chunk.chunk_index} to workspace${metaStr}\n\nLines ${chunk.start_line}-${chunk.end_line} — see <workspace_context> for content.\n\nWorkspace: ${cache.size} chunks, ${cache.totalTokens.toLocaleString()} tokens`
+    
+    // Save workspace asynchronously
+    cache.save().catch(() => {})
+  } catch (error) {
+    // Auto-attach failed — keep original output
+    // Silent failure (don't break read())
+  }
+}
+
+/**
+ * Substitute read() output if file has chunks in workspace (full reads only).
  *
- * Input: { filePath: "src/auth.ts", offset?: 0, limit?: 100 }
+ * Input: { filePath: "src/auth.ts" } (no offset/limit)
  * Output: "export function login(...)\n..."
  *
- * If file has chunks in workspace AND no offset/limit (full read):
+ * If file has chunks in workspace:
  *   Replace with: "[File "src/auth.ts" has N chunks in workspace (chunks: 2, 5, 7) — see <workspace_context>]"
- *
- * If offset/limit present (partial read):
- *   Keep original (partial reads are not in workspace injection)
  */
 function substituteReadOutput(output: { title: string; output: string; metadata: any }, cache: WorkspaceCache): void {
   try {
@@ -101,10 +261,6 @@ function substituteReadOutput(output: { title: string; output: string; metadata:
     const filePath = output.metadata?.filePath || extractFilePathFromTitle(output.title)
     if (!filePath) return
 
-    // Check if this is a partial read (offset/limit present)
-    const isPartialRead = output.metadata?.offset !== undefined || output.metadata?.limit !== undefined
-    if (isPartialRead) return
-
     // Don't substitute if file was modified (dirty) — workspace has stale content
     if (cache.isDirty(filePath)) return
 

package/index.ts

@@ -1,7 +1,7 @@
 import type { Plugin } from "@opencode-ai/plugin"
 
 import search from "./tools/search"
-import { workspace_list, workspace_attach, workspace_detach, workspace_clear, workspace_restore } from "./tools/workspace"
+import { workspace_list, workspace_forget, workspace_clear, workspace_restore } from "./tools/workspace"
 import FileIndexerPlugin from "./file-indexer"
 import { workspaceCache } from "./cache/manager"
 import { createWorkspaceInjectionHandler } from "./hooks/message-before"
@@ -39,8 +39,7 @@ const UsethisSearchPlugin: Plugin = async ({ directory, client }) => {
     tool: {
       search,
       workspace_list,
-      workspace_attach,
-      workspace_detach,
+      workspace_forget,
       workspace_clear,
       workspace_restore,
     },

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@comfanion/usethis_search",
-  "version": "4.2.0-dev.4",
-  "description": "OpenCode plugin: semantic search with chunk-based workspace injection (v4.2-dev: chunk-level context, granular detach, improved token efficiency)",
+  "version": "4.3.0-dev.1",
+  "description": "OpenCode plugin: semantic search with auto-attach, line numbers in workspace, simplified API (v4.3: auto-detect modes, read() caching, tool call compaction, 99% token reduction, no grep needed)",
   "type": "module",
   "main": "./index.ts",
   "exports": {
@@ -25,6 +25,7 @@
     "tools/search.ts",
     "tools/codeindex.ts",
     "tools/workspace.ts",
+    "tools/read-interceptor.ts",
     "cache/manager.ts",
     "hooks/message-before.ts",
     "hooks/tool-substitution.ts",

package/tools/read-interceptor.ts

@@ -0,0 +1,149 @@
+import { tool } from "@opencode-ai/plugin"
+import path from "path"
+import fs from "fs/promises"
+
+import { CodebaseIndexer } from "../vectorizer/index.ts"
+
+// FR-043: Logging for intercepted Read() calls
+const DEBUG = process.env.DEBUG?.includes("vectorizer") || process.env.DEBUG === "*"
+
+interface ReadLogEntry {
+  timestamp: number
+  filePath: string
+  relPath: string
+  chunksFound: number
+  relatedContextCount: number
+  durationMs: number
+  fallback: boolean
+}
+
+const LOG_MAX_ENTRIES = 500
+
+/**
+ * Append a log entry to the Read() interception log file.
+ * Non-blocking, non-fatal — errors are silently ignored.
+ */
+async function logReadInterception(projectRoot: string, entry: ReadLogEntry): Promise<void> {
+  try {
+    const logPath = path.join(projectRoot, ".opencode", "vectors", "read-intercept.log.json")
+    await fs.mkdir(path.dirname(logPath), { recursive: true })
+
+    let entries: ReadLogEntry[] = []
+    try {
+      const raw = await fs.readFile(logPath, "utf-8")
+      entries = JSON.parse(raw)
+    } catch {
+      // file doesn't exist or is invalid — start fresh
+    }
+
+    entries.push(entry)
+    // Cap log size to avoid unbounded growth
+    if (entries.length > LOG_MAX_ENTRIES) {
+      entries = entries.slice(-LOG_MAX_ENTRIES)
+    }
+
+    await fs.writeFile(logPath, JSON.stringify(entries, null, 2), "utf-8")
+  } catch {
+    // non-fatal — logging must never break Read
+  }
+}
+
+export default tool({
+  description: `Read file with graph-aware context attachment. When available, this tool searches the file in the index and returns content + related context from the graph (imports, links, etc.).
+
+Use this instead of the standard Read tool for better context awareness.`,
+
+  args: {
+    filePath: tool.schema.string().describe("Path to the file to read"),
+  },
+
+  async execute(args) {
+    const startTime = Date.now()
+    const projectRoot = process.cwd()
+    const filePath = path.isAbsolute(args.filePath) ? args.filePath : path.join(projectRoot, args.filePath)
+
+    const relPath = path.relative(projectRoot, filePath)
+
+    if (DEBUG) {
+      console.log(`[read-interceptor] Intercepted Read("${relPath}")`)
+    }
+
+    // Resilient search: if vector index is corrupted or unavailable, fall back gracefully
+    let fileChunks: any[] = []
+    let allRelated: any[] = []
+    let searchFailed = false
+
+    try {
+      const indexer = await new CodebaseIndexer(projectRoot, "code").init()
+      try {
+        const results = await indexer.search(relPath, 20, false, {})
+        fileChunks = results.filter((r: any) => r.file === relPath)
+
+        allRelated = fileChunks
+          .flatMap((c: any) => c.relatedContext || [])
+          .filter((r: any, i: number, arr: any[]) => arr.findIndex((x: any) => x.chunk_id === r.chunk_id) === i)
+      } catch (searchErr: any) {
+        if (DEBUG) {
+          console.log(`[read-interceptor] Search failed for "${relPath}": ${searchErr.message}`)
+        }
+        searchFailed = true
+      }
+      await indexer.unloadModel()
+    } catch (initErr: any) {
+      if (DEBUG) {
+        console.log(`[read-interceptor] Indexer init failed: ${initErr.message}`)
+      }
+      searchFailed = true
+    }
+
+    const durationMs = Date.now() - startTime
+    const fallback = fileChunks.length === 0
+
+    // FR-043: Log the interception asynchronously (non-blocking)
+    logReadInterception(projectRoot, {
+      timestamp: startTime,
+      filePath: args.filePath,
+      relPath,
+      chunksFound: fileChunks.length,
+      relatedContextCount: allRelated.length,
+      durationMs,
+      fallback,
+    }).catch(() => {})
+
+    if (DEBUG) {
+      console.log(
+        `[read-interceptor] ${relPath}: ${fileChunks.length} chunks, ${allRelated.length} related, ${durationMs}ms${fallback ? " (fallback)" : ""}${searchFailed ? " (search error)" : ""}`
+      )
+    }
+
+    if (fallback) {
+      const reason = searchFailed
+        ? `Search index unavailable (possibly corrupted). Run codeindex({ action: "reindex", index: "code" }) to rebuild.`
+        : `File "${relPath}" not indexed. Use original Read tool or run codeindex({ action: "reindex", index: "code" })`
+      return reason
+    }
+
+    let output = `## ${relPath}\n\n`
+
+    output += `### Content\n\n`
+    for (const chunk of fileChunks) {
+      output += chunk.content + "\n\n"
+    }
+
+    if (allRelated.length > 0) {
+      output += `### Related Context\n\n`
+      for (const rel of allRelated) {
+        const snippet = rel.content.length > 300
+          ? rel.content.substring(0, 300) + "..."
+          : rel.content
+        output += `**${rel.file}** (${rel.relation})\n`
+        output += `\`\`\`\n${snippet}\n\`\`\`\n\n`
+      }
+    }
+
+    return output
+  },
+})
+
+// Export for testing
+export { logReadInterception, ReadLogEntry }