@comfanion/usethis_search 4.1.0-dev.3 → 4.2.0-dev.1

package/tools/workspace.ts

@@ -1,24 +1,30 @@
 /**
- * Workspace Management Tools
+ * Workspace Management Tools (Chunk-Based)
  *
  * Manual control over the workspace cache:
- *   workspace_list    — show all attached files + stats
- *   workspace_attach  — manually attach a file by path
- *   workspace_detach  — remove file(s) from workspace
- *   workspace_clear   — remove all files
+ *   workspace_list    — show all attached chunks grouped by file
+ *   workspace_attach  — manually attach a file as single chunk
+ *   workspace_detach  — remove chunks by chunkId, path, query, or age
+ *   workspace_clear   — remove all chunks
  *   workspace_restore — restore a saved session snapshot
+ *
+ * Chunk-based architecture:
+ *   - Each file can be split into multiple chunks (e.g. "src/auth.ts:chunk-5")
+ *   - Chunks are keyed by chunkId, grouped by file for display
+ *   - Manual attach treats file as single chunk (chunkIndex=0, chunkId=path:chunk-0)
  */
 
 import { tool } from "@opencode-ai/plugin"
 import path from "path"
 import fs from "fs/promises"
+import crypto from "crypto"
 
 import { workspaceCache } from "../cache/manager.ts"
 
 // ── workspace.list ──────────────────────────────────────────────────────────
 
 export const workspace_list = tool({
-  description: `List all files currently in workspace context. Shows file paths, roles, scores, and token counts.`,
+  description: `List all chunks currently in workspace context, grouped by file. Shows chunk count, roles, scores, and token counts.`,
 
   args: {},
 
@@ -26,7 +32,7 @@ export const workspace_list = tool({
     const entries = workspaceCache.getAll()
 
     if (entries.length === 0) {
-      return `Workspace is empty.\n\nUse search() to find and attach files, or workspace.attach("path") to add manually.`
+      return `Workspace is empty.\n\nUse search() to find and attach chunks, or workspace_attach("path") to add a file manually.`
     }
 
     const sessionId = workspaceCache.getSessionId()
@@ -34,43 +40,84 @@ export const workspace_list = tool({
     if (sessionId) {
       output += `Session: ${sessionId}\n`
     }
-    output += `Files: ${workspaceCache.size}\n`
+    output += `Chunks: ${workspaceCache.size}\n`
     output += `Total tokens: ${workspaceCache.totalTokens.toLocaleString()}\n\n`
 
-    const mainFiles = entries.filter(e => e.role === "search-main")
-    const graphFiles = entries.filter(e => e.role === "search-graph")
-    const manualFiles = entries.filter(e => e.role === "manual")
+    // Group chunks by file
+    const fileGroups = new Map<string, typeof entries>()
+    for (const entry of entries) {
+      if (!fileGroups.has(entry.path)) {
+        fileGroups.set(entry.path, [])
+      }
+      fileGroups.get(entry.path)!.push(entry)
+    }
+
+    // Separate by role
+    const mainFiles = Array.from(fileGroups.entries()).filter(([_, chunks]) =>
+      chunks.some(c => c.role === "search-main")
+    )
+    const graphFiles = Array.from(fileGroups.entries()).filter(([_, chunks]) =>
+      chunks.some(c => c.role === "search-graph") && !chunks.some(c => c.role === "search-main")
+    )
+    const manualFiles = Array.from(fileGroups.entries()).filter(([_, chunks]) =>
+      chunks.some(c => c.role === "manual") && !chunks.some(c => c.role === "search-main" || c.role === "search-graph")
+    )
 
     if (mainFiles.length > 0) {
-      output += `### Search results (${mainFiles.length})\n`
-      for (const e of mainFiles) {
-        const age = Math.floor((Date.now() - e.attachedAt) / 1000 / 60)
-        const score = e.score ? ` score: ${e.score.toFixed(3)}` : ""
-        const meta = e.metadata?.function_name || e.metadata?.class_name || ""
-        output += `- **${e.path}** — ${e.tokens.toLocaleString()} tok${score}${meta ? ` (${meta})` : ""} — ${age}m ago\n`
-        if (e.attachedBy && e.attachedBy !== "manual") {
-          output += `  query: "${e.attachedBy}"\n`
+      output += `### Search results (${mainFiles.length} files)\n`
+      for (const [filePath, chunks] of mainFiles) {
+        const totalTokens = chunks.reduce((sum, c) => sum + c.tokens, 0)
+        const score = chunks[0]?.score ? ` score: ${chunks[0].score.toFixed(3)}` : ""
+        const meta = chunks[0]?.metadata?.function_name || chunks[0]?.metadata?.class_name || ""
+        const age = Math.floor((Date.now() - chunks[0].attachedAt) / 1000 / 60)
+        
+        output += `- **${filePath}** (${chunks.length} chunk${chunks.length > 1 ? "s" : ""}, ${totalTokens.toLocaleString()} tokens)${score}${meta ? ` (${meta})` : ""} — ${age}m ago\n`
+        
+        if (chunks.length > 1) {
+          for (const chunk of chunks) {
+            output += `  • ${chunk.chunkId} — chunk ${chunk.chunkIndex}, ${chunk.tokens.toLocaleString()} tok\n`
+          }
+        }
+        
+        if (chunks[0]?.attachedBy && chunks[0].attachedBy !== "manual") {
+          output += `  query: "${chunks[0].attachedBy}"\n`
         }
       }
       output += `\n`
     }
 
     if (graphFiles.length > 0) {
-      output += `### Graph relations (${graphFiles.length})\n`
-      for (const e of graphFiles) {
-        const age = Math.floor((Date.now() - e.attachedAt) / 1000 / 60)
-        const relation = e.metadata?.relation || "related"
-        const mainFile = e.metadata?.mainFile ? path.basename(e.metadata.mainFile) : "?"
-        output += `- **${e.path}** — ${e.tokens.toLocaleString()} tok — ${relation} from ${mainFile} — ${age}m ago\n`
+      output += `### Graph relations (${graphFiles.length} files)\n`
+      for (const [filePath, chunks] of graphFiles) {
+        const totalTokens = chunks.reduce((sum, c) => sum + c.tokens, 0)
+        const relation = chunks[0]?.metadata?.relation || "related"
+        const mainChunkId = chunks[0]?.metadata?.mainChunkId || "?"
+        const age = Math.floor((Date.now() - chunks[0].attachedAt) / 1000 / 60)
+        
+        output += `- **${filePath}** (${chunks.length} chunk${chunks.length > 1 ? "s" : ""}, ${totalTokens.toLocaleString()} tokens) — ${relation} from ${mainChunkId} — ${age}m ago\n`
+        
+        if (chunks.length > 1) {
+          for (const chunk of chunks) {
+            output += `  • ${chunk.chunkId} — chunk ${chunk.chunkIndex}, ${chunk.tokens.toLocaleString()} tok\n`
+          }
+        }
       }
       output += `\n`
     }
 
     if (manualFiles.length > 0) {
-      output += `### Manually attached (${manualFiles.length})\n`
-      for (const e of manualFiles) {
-        const age = Math.floor((Date.now() - e.attachedAt) / 1000 / 60)
-        output += `- **${e.path}** — ${e.tokens.toLocaleString()} tok — ${age}m ago\n`
+      output += `### Manually attached (${manualFiles.length} files)\n`
+      for (const [filePath, chunks] of manualFiles) {
+        const totalTokens = chunks.reduce((sum, c) => sum + c.tokens, 0)
+        const age = Math.floor((Date.now() - chunks[0].attachedAt) / 1000 / 60)
+        
+        output += `- **${filePath}** (${chunks.length} chunk${chunks.length > 1 ? "s" : ""}, ${totalTokens.toLocaleString()} tokens) — ${age}m ago\n`
+        
+        if (chunks.length > 1) {
+          for (const chunk of chunks) {
+            output += `  • ${chunk.chunkId} — chunk ${chunk.chunkIndex}, ${chunk.tokens.toLocaleString()} tok\n`
+          }
+        }
       }
       output += `\n`
     }
@@ -80,7 +127,7 @@ export const workspace_list = tool({
     const pct = Math.round((workspaceCache.totalTokens / config.maxTokens) * 100)
     output += `---\n`
     output += `*Budget: ${workspaceCache.totalTokens.toLocaleString()} / ${config.maxTokens.toLocaleString()} tokens (${pct}%) | `
-    output += `${workspaceCache.size} / ${config.maxFiles} files*`
+    output += `${workspaceCache.size} / ${config.maxChunks} chunks*`
 
     return output
   },
@@ -89,7 +136,7 @@ export const workspace_list = tool({
 // ── workspace.attach ────────────────────────────────────────────────────────
 
 export const workspace_attach = tool({
-  description: `Manually attach a file to workspace context. The file will be visible in context injection without needing read().`,
+  description: `Manually attach a file to workspace context as a single chunk. The file will be visible in context injection without needing read().`,
 
   args: {
     filePath: tool.schema.string().describe("Relative file path to attach (e.g. 'src/auth/login.ts')"),
@@ -98,26 +145,32 @@ export const workspace_attach = tool({
   async execute(args) {
     const projectRoot = process.cwd()
 
-    // Check if already attached
-    if (workspaceCache.has(args.filePath)) {
-      const entry = workspaceCache.get(args.filePath)!
-      return `File "${args.filePath}" is already in workspace.\nRole: ${entry.role} | Tokens: ${entry.tokens.toLocaleString()} | Score: ${entry.score?.toFixed(3) ?? "n/a"}`
-    }
-
     // Read file content
     try {
       const fullPath = path.join(projectRoot, args.filePath)
       const content = await fs.readFile(fullPath, "utf-8")
 
+      // Generate chunkId for manual attachment: "path:chunk-0"
+      const chunkId = `${args.filePath}:chunk-0`
+
+      // Check if already attached
+      if (workspaceCache.has(args.filePath)) {
+        const existing = workspaceCache.get(args.filePath)!
+        return `File "${args.filePath}" is already in workspace.\nChunkId: ${existing.chunkId} | Role: ${existing.role} | Tokens: ${existing.tokens.toLocaleString()} | Score: ${existing.score?.toFixed(3) ?? "n/a"}`
+      }
+
       workspaceCache.attach({
+        chunkId,
         path: args.filePath,
         content,
+        chunkIndex: 0,
         role: "manual",
         attachedAt: Date.now(),
         attachedBy: "manual",
       })
 
-      return `Attached "${args.filePath}" to workspace.\nTokens: ${workspaceCache.get(args.filePath)!.tokens.toLocaleString()}\nWorkspace total: ${workspaceCache.totalTokens.toLocaleString()} tokens (${workspaceCache.size} files)`
+      const entry = workspaceCache.get(args.filePath)!
+      return `Attached "${args.filePath}" to workspace as single chunk.\nChunkId: ${chunkId}\nTokens: ${entry.tokens.toLocaleString()}\nWorkspace total: ${workspaceCache.totalTokens.toLocaleString()} tokens (${workspaceCache.size} chunks)`
     } catch (error: any) {
       return `Failed to attach "${args.filePath}": ${error.message || String(error)}`
     }
@@ -127,38 +180,60 @@ export const workspace_attach = tool({
 // ── workspace.detach ────────────────────────────────────────────────────────
 
 export const workspace_detach = tool({
-  description: `Remove file(s) from workspace context. Can detach by path, by search query, or by age.`,
+  description: `Remove chunks from workspace context. Can detach by chunkId, by file path (removes ALL chunks of that file), by search query, or by age.`,
 
   args: {
-    filePath: tool.schema.string().optional().describe("Specific file path to remove"),
-    query: tool.schema.string().optional().describe("Remove all files attached by this search query"),
-    olderThan: tool.schema.number().optional().describe("Remove files older than N minutes"),
+    chunkId: tool.schema.string().optional().describe("Specific chunk ID to remove (e.g. 'src/auth.ts:chunk-5')"),
+    filePath: tool.schema.string().optional().describe("File path to remove (removes ALL chunks of that file)"),
+    query: tool.schema.string().optional().describe("Remove all chunks attached by this search query"),
+    olderThan: tool.schema.number().optional().describe("Remove chunks older than N minutes"),
   },
 
   async execute(args) {
     let removed = 0
 
-    if (args.filePath) {
-      removed = workspaceCache.detach(args.filePath) ? 1 : 0
+    if (args.chunkId) {
+      // Detach specific chunk by chunkId
+      const entries = workspaceCache.getAll()
+      const entry = entries.find(e => e.chunkId === args.chunkId)
+      
+      if (!entry) {
+        return `Chunk "${args.chunkId}" not found in workspace.`
+      }
+      
+      removed = workspaceCache.detach(entry.path) ? 1 : 0
       if (removed === 0) {
+        return `Failed to remove chunk "${args.chunkId}".`
+      }
+    } else if (args.filePath) {
+      // Detach all chunks of a file
+      const entries = workspaceCache.getAll()
+      const fileChunks = entries.filter(e => e.path === args.filePath)
+      
+      if (fileChunks.length === 0) {
         return `File "${args.filePath}" not found in workspace.`
       }
+      
+      removed = workspaceCache.detach(args.filePath) ? fileChunks.length : 0
+      if (removed === 0) {
+        return `Failed to remove chunks from "${args.filePath}".`
+      }
     } else if (args.query) {
       removed = workspaceCache.detachByQuery(args.query)
     } else if (args.olderThan) {
       removed = workspaceCache.detachOlderThan(args.olderThan * 60 * 1000)
     } else {
-      return `Specify filePath, query, or olderThan to detach files.`
+      return `Specify chunkId, filePath, query, or olderThan to detach chunks.`
     }
 
-    return `Removed ${removed} file(s) from workspace.\nWorkspace: ${workspaceCache.size} files, ${workspaceCache.totalTokens.toLocaleString()} tokens`
+    return `Removed ${removed} chunk(s) from workspace.\nWorkspace: ${workspaceCache.size} chunks, ${workspaceCache.totalTokens.toLocaleString()} tokens`
   },
 })
 
 // ── workspace.clear ─────────────────────────────────────────────────────────
 
 export const workspace_clear = tool({
-  description: `Remove ALL files from workspace context. Use when switching tasks or starting fresh.`,
+  description: `Remove ALL chunks from workspace context. Use when switching tasks or starting fresh.`,
 
   args: {},
 
@@ -167,7 +242,7 @@ export const workspace_clear = tool({
     const tokens = workspaceCache.totalTokens
     workspaceCache.clear()
 
-    return `Cleared workspace: ${count} files removed (${tokens.toLocaleString()} tokens freed).\nWorkspace is now empty.`
+    return `Cleared workspace: ${count} chunks removed (${tokens.toLocaleString()} tokens freed).\nWorkspace is now empty.`
   },
 })
 
@@ -192,9 +267,9 @@ export const workspace_restore = tool({
       let output = `## Saved Workspace Snapshots\n\n`
       for (const snap of snapshots) {
         const date = new Date(snap.savedAt).toLocaleString()
-        output += `- **${snap.id}** — ${snap.fileCount} files, ${snap.totalTokens.toLocaleString()} tokens — ${date}\n`
+        output += `- **${snap.id}** — ${snap.chunkCount} chunks, ${snap.totalTokens.toLocaleString()} tokens — ${date}\n`
       }
-      output += `\nUse \`workspace.restore("session-id")\` to restore.`
+      output += `\nUse \`workspace_restore("session-id")\` to restore.`
       return output
     }
 
@@ -205,6 +280,6 @@ export const workspace_restore = tool({
       return `Snapshot "${args.sessionId}" not found or empty.`
     }
 
-    return `Restored workspace from "${args.sessionId}".\nFiles: ${workspaceCache.size}\nTokens: ${workspaceCache.totalTokens.toLocaleString()}`
+    return `Restored workspace from "${args.sessionId}".\nChunks: ${workspaceCache.size}\nTokens: ${workspaceCache.totalTokens.toLocaleString()}`
   },
 })

package/vectorizer.yaml

@@ -55,17 +55,17 @@ vectorizer:
     # Read() intercept
     read_intercept: true
 
-  # Workspace injection (v4) — search results attached to AI context
+  # Workspace injection (v4.1) — chunk-based context
   workspace:
-    max_tokens: 50000            # Max total tokens across all cached files
-    max_files: 30                # Max number of files in workspace
-    attach_top_n: 5              # Top N search results to attach with full content
-    attach_related_per_file: 3   # Max graph relations per main file
-    min_score_main: 0.65         # Min score for main files
-    min_score_related: 0.5       # Min score for graph relations
-    persist_content: false       # Save full content in snapshots (debug mode)
-    auto_prune_search: true      # Replace old search outputs with compact summaries
-    substitute_tool_outputs: true # Replace read() outputs when file in workspace
+    max_tokens: 50000              # Max total tokens across all cached chunks
+    max_chunks: 100                # Max number of chunks in workspace
+    attach_top_n: 10               # Top N search chunks to attach with full content
+    attach_related_per_chunk: 3    # Max graph relation chunks per main chunk
+    min_score_main: 0.65           # Min score for main chunks
+    min_score_related: 0.5         # Min score for graph relation chunks
+    persist_content: false         # Save full chunk content in snapshots (debug mode)
+    auto_prune_search: true        # Replace old search outputs with compact summaries
+    substitute_tool_outputs: true  # Replace read() outputs when chunks in workspace
 
   # Quality monitoring (v2)
   quality: