@ridit/lens 0.2.1 → 0.2.2

package/src/index.tsx

@@ -7,6 +7,9 @@ import { ReviewCommand } from "./commands/review";
 import { TaskCommand } from "./commands/task";
 import { ChatCommand } from "./commands/chat";
 import { TimelineCommand } from "./commands/timeline";
+import { registerBuiltins } from "./utils/tools/builtins";
+
+registerBuiltins();
 
 const program = new Command();
 

package/src/prompts/system.ts

@@ -3,63 +3,19 @@ import type { ImportantFile } from "../types/repo";
 export function buildSystemPrompt(
   files: ImportantFile[],
   memorySummary = "",
+  toolsSection?: string,
 ): string {
   const fileList = files
     .map((f) => `### ${f.path}\n\`\`\`\n${f.content.slice(0, 2000)}\n\`\`\``)
     .join("\n\n");
 
-  return `You are an expert software engineer assistant with access to the user's codebase and tools.
-
-## TOOLS
-
-You have exactly thirteen tools. To use a tool you MUST wrap it in the exact XML tags shown below — no other format will work.
-
-### 1. fetch — load a URL
-<fetch>https://example.com</fetch>
+  // If a toolsSection is supplied (e.g. from the plugin registry), use it.
+  // Otherwise fall back to the static built-in section.
+  const tools = toolsSection ?? BUILTIN_TOOLS_SECTION;
 
-### 2. shell — run a terminal command
-<shell>node -v</shell>
-
-### 3. read-file — read a file from the repo
-<read-file>src/foo.ts</read-file>
-
-### 4. read-folder — list contents of a folder (files + subfolders, one level deep)
-<read-folder>src/components</read-folder>
-
-### 5. grep — search for a pattern across files in the repo (cross-platform, no shell needed)
-<grep>
-{"pattern": "ChatRunner", "glob": "src/**/*.tsx"}
-</grep>
-
-### 6. write-file — create or overwrite a file
-<write-file>
-{"path": "data/output.csv", "content": "col1,col2\nval1,val2"}
-</write-file>
-
-### 7. delete-file — permanently delete a single file
-<delete-file>src/old-component.tsx</delete-file>
-
-### 8. delete-folder — permanently delete a folder and all its contents
-<delete-folder>src/legacy</delete-folder>
-
-### 9. open-url — open a URL in the user's default browser
-<open-url>https://github.com/owner/repo</open-url>
-
-### 10. generate-pdf — generate a PDF file from markdown-style content
-<generate-pdf>
-{"path": "output/report.pdf", "content": "# Title\n\nSome body text.\n\n## Section\n\nMore content."}
-</generate-pdf>
-
-### 11. search — search the internet for anything you are unsure about
-<search>how to use React useEffect cleanup function</search>
-
-### 12. clone — clone a GitHub repo so you can explore and discuss it
-<clone>https://github.com/owner/repo</clone>
+  return `You are an expert software engineer assistant with access to the user's codebase and tools.
 
-### 13. changes — propose code edits (shown as a diff for user approval)
-<changes>
-{"summary": "what changed and why", "patches": [{"path": "src/foo.ts", "content": "COMPLETE file content", "isNew": false}]}
-</changes>
+${tools}
 
 ## MEMORY OPERATIONS
 
@@ -109,9 +65,26 @@ You may emit multiple memory operations in a single response alongside normal co
 22. When scaffolding a multi-file project, after each write-file succeeds, immediately proceed to writing the NEXT file — NEVER rewrite a file you already wrote in this session. Each file is written ONCE and ONLY ONCE.
 23. For JSX/TSX files always use \`.tsx\` extension and include \`/** @jsxImportSource react */\` or ensure tsconfig has jsx set — bun needs this to parse JSX
 24. When explaining how to use a tool in text, use [tag] bracket notation or a fenced code block — NEVER emit a real XML tool tag as part of an explanation or example
-25. NEVER chain tool calls unless the user's request explicitly requires multiple steps
-26. NEVER read files, list folders, or run tools that were not asked for in the current user message
-27. NEVER use markdown formatting in plain text responses — no **bold**, no *italics*, no # headings, no bullet points with -, *, or +, no numbered lists, no backtick inline code. Write in plain prose. Only use fenced \`\`\` code blocks when showing actual code.
+25. NEVER read files, list folders, or run tools that were not asked for in the current user message
+26. NEVER use markdown formatting in plain text responses — no **bold**, no *italics*, no # headings, no bullet points with -, *, or +, no numbered lists, no backtick inline code. Write in plain prose. Only use fenced \`\`\` code blocks when showing actual code.
+
+## SCAFFOLDING — CHAINING WRITE-FILE CALLS
+
+When creating multiple files (e.g. scaffolding a project or creating 10 files), emit ALL of them
+in a single response by chaining the tags back-to-back with no text between them:
+
+<write-file>
+{"path": "test/file1.txt", "content": "File 1 content"}
+</write-file>
+<write-file>
+{"path": "test/file2.txt", "content": "File 2 content"}
+</write-file>
+<write-file>
+{"path": "test/file3.txt", "content": "File 3 content"}
+</write-file>
+
+The system processes each tag sequentially and automatically continues to the next one.
+Do NOT wait for a user message between files — emit all tags at once.
 
 ## CRITICAL: READ BEFORE YOU WRITE
 
@@ -133,58 +106,62 @@ These rules are mandatory whenever you plan to edit or create a file:
 - NEVER produce a file that is shorter than the original unless you are explicitly asked to delete things
 - If you catch yourself rewriting a file from scratch, STOP — go back and read the original first
 
-## WHEN TO USE read-folder:
-- Before editing files in an unfamiliar directory — list it first to understand the structure
-- When a feature spans multiple files and you are not sure what exists
-- When the user asks you to explore or explain a part of the codebase
-
-## SCAFFOLDING A NEW PROJECT (follow this exactly)
-
-When the user asks to create a new CLI/app in a subfolder (e.g. "make a todo app called list"):
-1. Create all files first using write-file with paths like \`list/package.json\`, \`list/src/index.tsx\`
-2. Then run \`cd list && bun install\` (or npm/pnpm) in one shell command
-3. Then run the project with \`cd list && bun run index.ts\` or whatever the entry point is
-4. NEVER run \`bun init\` — it is interactive and will hang. Create package.json manually with write-file instead
-5. TSX files need either tsconfig.json with \`"jsx": "react-jsx"\` or \`/** @jsxImportSource react */\` at the top
-
-## FETCH → WRITE FLOW (follow this exactly when saving fetched data)
-
-1. fetch the URL
-2. Analyze the result — count the rows, identify columns, check completeness
-3. Tell the user what you found: "Found X rows with columns: A, B, C. Writing now."
-4. emit write-file with correctly structured, complete content
-5. After write-file confirms success, emit read-file to verify
-6. Only after read-file confirms content is correct, tell the user it is done
-
-## WHEN TO USE TOOLS
-
-- User shares any URL → fetch it immediately
-- User asks to run anything → shell it immediately
-- User asks to open a link, open a URL, or visit a website → open-url it immediately, do NOT use fetch
-- User asks to delete a file → delete-file it immediately (requires approval)
-- User asks to delete a folder or directory → delete-folder it immediately (requires approval)
-- User asks to search for a pattern in files, find usages, find where something is defined → grep it immediately, NEVER use shell grep/findstr/Select-String
-- User asks to read a file → read-file it immediately, NEVER use shell cat/type
-- User asks what files are in a folder, or to explore/list a directory → read-folder it immediately, NEVER use shell ls/dir/find/git ls-files
-- User asks to explore a folder or directory → read-folder it immediately
-- User asks to save/create/write a file → write-file it immediately, then read-file to verify
-- User asks to modify/edit/add to an existing file → read-file it FIRST, then emit changes
-- User shares a GitHub URL and wants to clone/explore/discuss it → use clone immediately, NEVER use shell git clone
-- After clone succeeds, you will see context about the clone in the conversation. Wait for the user to ask a specific question before using any tools. Do NOT auto-read files, do NOT emit any tool tags until the user asks.
-- You are unsure about an API, library, error, concept, or piece of code → search it immediately
-- User asks about something recent or that you might not know → search it immediately
-- You are about to say "I'm not sure" or "I don't know" → search instead of guessing
-
-## shell IS ONLY FOR:
-- Running code: \`node script.js\`, \`bun run dev\`, \`python main.py\`
-- Installing packages: \`npm install\`, \`pip install\`
-- Building/testing: \`npm run build\`, \`bun test\`
-- Git operations other than clone: \`git status\`, \`git log\`, \`git diff\`
-- Anything that EXECUTES — not reads or lists
-
 ## CODEBASE
 
 ${fileList.length > 0 ? fileList : "(no files indexed)"}
 
 ${memorySummary}`;
 }
+
+// ── Static fallback tools section (used when registry is not available) ───────
+
+const BUILTIN_TOOLS_SECTION = `## TOOLS
+
+You have exactly thirteen tools. To use a tool you MUST wrap it in the exact XML tags shown below — no other format will work.
+
+### 1. fetch — load a URL
+<fetch>https://example.com</fetch>
+
+### 2. shell — run a terminal command
+<shell>node -v</shell>
+
+### 3. read-file — read a file from the repo
+<read-file>src/foo.ts</read-file>
+
+### 4. read-folder — list contents of a folder (files + subfolders, one level deep)
+<read-folder>src/components</read-folder>
+
+### 5. grep — search for a pattern across files in the repo (cross-platform, no shell needed)
+<grep>
+{"pattern": "ChatRunner", "glob": "src/**/*.tsx"}
+</grep>
+
+### 6. write-file — create or overwrite a file
+<write-file>
+{"path": "data/output.csv", "content": "col1,col2\\nval1,val2"}
+</write-file>
+
+### 7. delete-file — permanently delete a single file
+<delete-file>src/old-component.tsx</delete-file>
+
+### 8. delete-folder — permanently delete a folder and all its contents
+<delete-folder>src/legacy</delete-folder>
+
+### 9. open-url — open a URL in the user's default browser
+<open-url>https://github.com/owner/repo</open-url>
+
+### 10. generate-pdf — generate a PDF file from markdown-style content
+<generate-pdf>
+{"path": "output/report.pdf", "content": "# Title\\n\\nSome body text.\\n\\n## Section\\n\\nMore content."}
+</generate-pdf>
+
+### 11. search — search the internet for anything you are unsure about
+<search>how to use React useEffect cleanup function</search>
+
+### 12. clone — clone a GitHub repo so you can explore and discuss it
+<clone>https://github.com/owner/repo</clone>
+
+### 13. changes — propose code edits (shown as a diff for user approval)
+<changes>
+{"summary": "what changed and why", "patches": [{"path": "src/foo.ts", "content": "COMPLETE file content", "isNew": false}]}
+</changes>`;

package/src/utils/chat.ts

@@ -1,3 +1,9 @@
+// ── chat.ts ───────────────────────────────────────────────────────────────────
+//
+// Response parsing and API calls.
+// Tool parsing is now fully driven by the ToolRegistry — adding a new tool
+// to the registry automatically makes it parseable here.
+
 export {
   walkDir,
   applyPatches,
@@ -13,104 +19,87 @@ export { fetchUrl, searchWeb } from "../tools/web";
 export { generatePdf } from "../tools/pdf";
 export { buildSystemPrompt, FEW_SHOT_MESSAGES } from "../prompts";
 
-import type { FilePatch } from "../components/repo/DiffViewer";
 import type { Message } from "../types/chat";
 import type { Provider } from "../types/config";
 import { FEW_SHOT_MESSAGES } from "../prompts";
-
-// ── Response parser ───────────────────────────────────────────────────────────
+import { registry } from "./tools/registry";
+import type { FilePatch } from "../components/repo/DiffViewer";
 
 export type ParsedResponse =
-  | { kind: "text"; content: string }
-  | { kind: "changes"; content: string; patches: FilePatch[] }
-  | { kind: "shell"; content: string; command: string }
-  | { kind: "fetch"; content: string; url: string }
-  | { kind: "read-file"; content: string; filePath: string }
-  | { kind: "read-folder"; content: string; folderPath: string }
-  | { kind: "grep"; content: string; pattern: string; glob: string }
-  | { kind: "delete-file"; content: string; filePath: string }
-  | { kind: "delete-folder"; content: string; folderPath: string }
-  | { kind: "open-url"; content: string; url: string }
+  | { kind: "text"; content: string; remainder?: string }
   | {
-      kind: "generate-pdf";
+      kind: "changes";
       content: string;
-      filePath: string;
-      pdfContent: string;
+      patches: FilePatch[];
+      remainder?: string;
     }
+  | { kind: "clone"; content: string; repoUrl: string; remainder?: string }
   | {
-      kind: "write-file";
+      kind: "tool";
+      toolName: string;
+      input: unknown;
+      rawInput: string;
       content: string;
-      filePath: string;
-      fileContent: string;
-    }
-  | { kind: "search"; content: string; query: string }
-  | { kind: "clone"; content: string; repoUrl: string };
+      remainder?: string;
+    };
 
 export function parseResponse(text: string): ParsedResponse {
   const scanText = text.replace(/```[\s\S]*?```/g, (m) => " ".repeat(m.length));
 
   type Candidate = {
     index: number;
-    kind:
-      | "changes"
-      | "shell"
-      | "fetch"
-      | "read-file"
-      | "read-folder"
-      | "grep"
-      | "delete-file"
-      | "delete-folder"
-      | "open-url"
-      | "generate-pdf"
-      | "write-file"
-      | "search"
-      | "clone";
+    toolName: string;
     match: RegExpExecArray;
   };
   const candidates: Candidate[] = [];
 
-  const patterns: { kind: Candidate["kind"]; re: RegExp }[] = [
-    { kind: "fetch", re: /<fetch>([\s\S]*?)<\/fetch>/g },
-    { kind: "shell", re: /<shell>([\s\S]*?)<\/shell>/g },
-    { kind: "read-file", re: /<read-file>([\s\S]*?)<\/read-file>/g },
-    { kind: "read-folder", re: /<read-folder>([\s\S]*?)<\/read-folder>/g },
-    { kind: "grep", re: /<grep>([\s\S]*?)<\/grep>/g },
-    { kind: "delete-file", re: /<delete-file>([\s\S]*?)<\/delete-file>/g },
-    {
-      kind: "delete-folder",
-      re: /<delete-folder>([\s\S]*?)<\/delete-folder>/g,
-    },
-    { kind: "open-url", re: /<open-url>([\s\S]*?)<\/open-url>/g },
-    { kind: "generate-pdf", re: /<generate-pdf>([\s\S]*?)<\/generate-pdf>/g },
-    { kind: "write-file", re: /<write-file>([\s\S]*?)<\/write-file>/g },
-    { kind: "search", re: /<search>([\s\S]*?)<\/search>/g },
-    { kind: "clone", re: /<clone>([\s\S]*?)<\/clone>/g },
-    { kind: "changes", re: /<changes>([\s\S]*?)<\/changes>/g },
-    // fenced fallbacks
-    { kind: "fetch", re: /```fetch\r?\n([\s\S]*?)\r?\n```/g },
-    { kind: "shell", re: /```shell\r?\n([\s\S]*?)\r?\n```/g },
-    { kind: "read-file", re: /```read-file\r?\n([\s\S]*?)\r?\n```/g },
-    { kind: "read-folder", re: /```read-folder\r?\n([\s\S]*?)\r?\n```/g },
-    { kind: "write-file", re: /```write-file\r?\n([\s\S]*?)\r?\n```/g },
-    { kind: "search", re: /```search\r?\n([\s\S]*?)\r?\n```/g },
-    { kind: "changes", re: /```changes\r?\n([\s\S]*?)\r?\n```/g },
-  ];
+  for (const toolName of registry.names()) {
+    const escaped = toolName.replace(/[-]/g, "\\-");
+
+    // XML tag
+    const xmlRe = new RegExp(`<${escaped}>([\\s\\S]*?)<\\/${escaped}>`, "g");
+    xmlRe.lastIndex = 0;
+    const xmlM = xmlRe.exec(scanText);
+    if (xmlM) {
+      const orig = new RegExp(xmlRe.source);
+      const origM = orig.exec(text.slice(xmlM.index));
+      if (origM) {
+        candidates.push({
+          index: xmlM.index,
+          toolName,
+          match: Object.assign(
+            [
+              text.slice(xmlM.index, xmlM.index + origM[0].length),
+              origM[1],
+            ] as unknown as RegExpExecArray,
+            { index: xmlM.index, input: text, groups: undefined },
+          ),
+        });
+      }
+    }
 
-  for (const { kind, re } of patterns) {
-    re.lastIndex = 0;
-    const m = re.exec(scanText);
-    if (m) {
-      const originalRe = new RegExp(re.source, re.flags.replace("g", ""));
-      const originalMatch = originalRe.exec(text.slice(m.index));
-      if (originalMatch) {
-        const fakeMatch = Object.assign(
-          [
-            text.slice(m.index, m.index + originalMatch[0].length),
-            originalMatch[1],
-          ] as unknown as RegExpExecArray,
-          { index: m.index, input: text, groups: undefined },
-        );
-        candidates.push({ index: m.index, kind, match: fakeMatch });
+    // Fenced code block fallback
+    const fencedRe = new RegExp(
+      `\`\`\`${escaped}\\r?\\n([\\s\\S]*?)\\r?\\n\`\`\``,
+      "g",
+    );
+    fencedRe.lastIndex = 0;
+    const fencedM = fencedRe.exec(scanText);
+    if (fencedM) {
+      const orig = new RegExp(fencedRe.source);
+      const origM = orig.exec(text.slice(fencedM.index));
+      if (origM) {
+        candidates.push({
+          index: fencedM.index,
+          toolName,
+          match: Object.assign(
+            [
+              text.slice(fencedM.index, fencedM.index + origM[0].length),
+              origM[1],
+            ] as unknown as RegExpExecArray,
+            { index: fencedM.index, input: text, groups: undefined },
+          ),
+        });
       }
     }
   }
@@ -118,104 +107,56 @@ export function parseResponse(text: string): ParsedResponse {
   if (candidates.length === 0) return { kind: "text", content: text.trim() };
 
   candidates.sort((a, b) => a.index - b.index);
-  const { kind, match } = candidates[0]!;
+  const { toolName, match } = candidates[0]!;
 
-  const before = text
-    .slice(0, match.index)
-    .replace(
-      /<(fetch|shell|read-file|read-folder|write-file|search|clone|changes)[^>]*>[\s\S]*?<\/\1>/g,
-      "",
-    )
-    .trim();
+  const before = text.slice(0, match.index).trim();
   const body = (match[1] ?? "").trim();
+  const afterMatch = text.slice(match.index + match[0].length).trim();
+  const remainder = afterMatch.length > 0 ? afterMatch : undefined;
 
-  if (kind === "changes") {
+  // Special UI variants
+  if (toolName === "changes") {
     try {
       const parsed = JSON.parse(body) as {
         summary: string;
         patches: FilePatch[];
       };
       const display = [before, parsed.summary].filter(Boolean).join("\n\n");
-      return { kind: "changes", content: display, patches: parsed.patches };
+      return {
+        kind: "changes",
+        content: display,
+        patches: parsed.patches,
+        remainder,
+      };
     } catch {
-      /* fall through */
+      return { kind: "text", content: text.trim() };
     }
   }
 
-  if (kind === "shell")
-    return { kind: "shell", content: before, command: body };
-  if (kind === "fetch")
-    return {
-      kind: "fetch",
-      content: before,
-      url: body.replace(/^<|>$/g, "").trim(),
-    };
-  if (kind === "read-file")
-    return { kind: "read-file", content: before, filePath: body };
-  if (kind === "read-folder")
-    return { kind: "read-folder", content: before, folderPath: body };
-  if (kind === "delete-file")
-    return { kind: "delete-file", content: before, filePath: body };
-  if (kind === "delete-folder")
-    return { kind: "delete-folder", content: before, folderPath: body };
-  if (kind === "open-url")
-    return {
-      kind: "open-url",
-      content: before,
-      url: body.replace(/^<|>$/g, "").trim(),
-    };
-  if (kind === "search")
-    return { kind: "search", content: before, query: body };
-  if (kind === "clone")
+  if (toolName === "clone") {
     return {
       kind: "clone",
       content: before,
       repoUrl: body.replace(/^<|>$/g, "").trim(),
+      remainder,
     };
-
-  if (kind === "generate-pdf") {
-    try {
-      const parsed = JSON.parse(body);
-      return {
-        kind: "generate-pdf",
-        content: before,
-        filePath: parsed.path ?? parsed.filePath ?? "output.pdf",
-        pdfContent: parsed.content ?? "",
-      };
-    } catch {
-      return { kind: "text", content: text };
-    }
   }
 
-  if (kind === "grep") {
-    try {
-      const parsed = JSON.parse(body) as { pattern: string; glob?: string };
-      return {
-        kind: "grep",
-        content: before,
-        pattern: parsed.pattern,
-        glob: parsed.glob ?? "**/*",
-      };
-    } catch {
-      return { kind: "grep", content: before, pattern: body, glob: "**/*" };
-    }
-  }
+  // Generic tool
+  const tool = registry.get(toolName);
+  if (!tool) return { kind: "text", content: text.trim() };
 
-  if (kind === "write-file") {
-    try {
-      const parsed = JSON.parse(body) as { path: string; content: string };
-      return {
-        kind: "write-file",
-        content: before,
-        filePath: parsed.path,
-        fileContent: parsed.content,
-      };
-    } catch {
-      /* fall through */
-    }
-  }
+  const input = tool.parseInput(body);
+  if (input === null) return { kind: "text", content: text.trim() };
 
-  return { kind: "text", content: text.trim() };
+  return {
+    kind: "tool",
+    toolName,
+    input,
+    rawInput: body,
+    content: before,
+    remainder,
+  };
 }
 
 // ── Clone tag helper ──────────────────────────────────────────────────────────
@@ -251,31 +192,9 @@ function buildApiMessages(
             "The tool call was denied by the user. Please respond without using that tool.",
         };
       }
-      const label =
-        m.toolName === "shell"
-          ? `shell command \`${m.content}\``
-          : m.toolName === "fetch"
-            ? `fetch of ${m.content}`
-            : m.toolName === "read-file"
-              ? `read-file of ${m.content}`
-              : m.toolName === "read-folder"
-                ? `read-folder of ${m.content}`
-                : m.toolName === "grep"
-                  ? `grep for "${m.content}"`
-                  : m.toolName === "delete-file"
-                    ? `delete-file of ${m.content}`
-                    : m.toolName === "delete-folder"
-                      ? `delete-folder of ${m.content}`
-                      : m.toolName === "open-url"
-                        ? `open-url ${m.content}`
-                        : m.toolName === "generate-pdf"
-                          ? `generate-pdf to ${m.content}`
-                          : m.toolName === "search"
-                            ? `web search for "${m.content}"`
-                            : `write-file to ${m.content}`;
       return {
         role: "user",
-        content: `Here is the output from the ${label}:\n\n${m.result}\n\nPlease continue your response based on this output.`,
+        content: `Here is the output from the ${m.toolName} of ${m.content}:\n\n${m.result}\n\nPlease continue your response based on this output.`,
       };
     }
     return { role: m.role, content: m.content };