decorated-pi 0.4.1 → 0.5.1

package/README.md

@@ -1,6 +1,6 @@
 # decorated-pi
 
-`decorated-pi` is a practical enhancement pack for [Pi](https://github.com/earendil-works/pi).
+`decorated-pi` is a practical enhancement pack for [Pi](https://github.com/earendil-works/pi) — smarter tools that are token efficient and cache friendly.
 
 ## Install
 
@@ -14,67 +14,60 @@ pi install /path/to/decorated-pi
 
 ### 1. Patch Tool
 
-Replaces Pi's built-in `edit` / `write` with a stronger `patch` tool:
+Replaces Pi's built-in `edit` / `write` with a stronger `patch` tool that adds unique safety and usability improvements on top of the native tools.
 
-- **anchor mechanism** — narrows the search range by specifying a unique string that appears before `old_str`, preventing mismatches in files with repeated patterns
-- **mtime tracking** — records file modification time on `read`, rejects `patch` if the file changed since last read, preventing blind or stale edits
-- **explicit overwrite** — offer atomic `overwrite: true` mode for overwrite files or full-file creation to prevent unintened overwrite
+| Capability | Pi native `edit` | `patch` |
+| ------ | :---: | :---: |
+| Exact string replacement | ✅ `oldText` | ✅ `old_str` |
+| Atomic overwrite | ✅ `write` | ✅ `overwrite` |
+| Syntax‑highlighted overwrite | ✅ streaming | ✅ incremental |
+| **Anchor‑based search** | ❌ extending `oldText` for uniqueness | ✅ `anchor` bounds scope for precise matching |
+| **Fuzzy whitespace match** | ❌ only reports "not found" | ✅ auto‑corrects tab↔space / trailing whitespace mismatches |
+| **Edit fault diagnostics** | ❌ only reports "not found" | ✅ pinpoint faults for LLM comprehension |
+| **Stale‑read protection** | ❌ Blind to external changes | ✅ `read` captures mtime, `patch` rejects stale targets |
 
-### 2. Secret redaction
+### 2. Smarter `@` File Search
 
-Three-layer detection: high-confidence known-format patterns (AWS, GitHub, OpenAI, etc.), config-key regex matching, and adjusted Shannon entropy heuristics for unknown secret-like values.
-
-### 3. Built-in MCP Client
+Replaces Pi's built-in `@` file completion with smarter matching and noise filtering:
 
-Zero-config MCP client with two built-in servers:
+| Aspect | Pi native `@` | `decorated-pi` |
+| ------ | :---: | :---: |
+| **Speed** | ❌ re‑scans filesystem on every trigger | ✅ caches once per `@` trigger |
+| **Noise filtering** | ❌ no penalty system, shows hidden files | ✅ tiered penalty auto‑filters clutter |
+| **Default suggestions** | ❌ all files visible on empty query | ✅ only visible project files |
+| **Match precision** | ❌ case‑insensitive simple scoring | ✅ multi‑level case‑sensitive scoring |
 
-| Server | Tool Prefix | Source |
-| --- | --- | --- |
-| Context7 | `context7_*` | `https://mcp.context7.com/mcp` |
-| Exa | `exa_*` | `https://mcp.exa.ai/mcp` |
+### 3. Secret redaction
 
-**Project-level config** — add custom MCP servers by placing an `mcp.json` or `.mcp.json` (also `.pi/mcp.json`, `.agents/mcp.json`, `.claude/mcp.json` and their `.`-prefixed variants) anywhere in your project tree. Servers found closer to `cwd` take precedence over ancestor directories.
-
-```json
-{
-  "mcpServers": {
-    "my-server": {
-      "url": "https://my-mcp.example.com/mcp",
-      "enabled": true
-    }
-  }
-}
-```
+Three-layer detection: high-confidence known-format patterns (AWS, GitHub, OpenAI, etc.), config-key regex matching, and adjusted Shannon entropy heuristics for unknown secret-like values.
 
-**Global config** — persist custom servers in `~/.pi/agent/decorated-pi.json`:
+Example redaction on a `read` / `bash` output:
 
 ```json
 {
-  "mcpServers": {
-    "my-server": { "url": "https://my-mcp.example.com/mcp" }
-  }
+  "aws_access_key_id": "AKI**************PLE",
+  "github_token": "ghp***************def",
+  "database_password": "Sup#######t99",
+  "api_key": "sk_**************f5a",
+  "random_secret": "a1b??????5f5"
 }
 ```
 
-**Priority**: project > global > builtin. A server with the same name from a higher-priority source overrides lower ones.
-
-Use `/mcp` to view connection status and registered tools.
+> `*` = known pattern, `#` = config key regex, `?` = entropy heuristic.
 
-### 4. Auxiliary Models (Image + Compact)
+### 4. Auxiliary Models
 
-Uses cheaper models for auxiliary tasks, configured via `/dp-model`:
+Offloads auxiliary ops to cheaper models, reducing cost on every session. Configured via `/dp-model`:
 
 - **Image read fallback** — when the model reads an image file, detects type via magic bytes, calls a configured vision-capable model, and replaces the read result with image analysis text (jpeg, png, gif, webp)
-- **Compact model** — uses a configured model for context compaction (instead of the main model), auto-resumes after compaction.
+- **Compact model** — uses a configured model for context compaction (instead of the main model).
 
-### 5. Smart `@` File Search
+### 5. Progressive Context from `AGENTS.md` / `CLAUDE.md`
 
-Replaces Pi's default file search with a faster, project-aware search strategy:
+Extension capability: context is disclosed progressively as the agent works across different parts of the project.
 
-- Uses project-aware file discovery
-- Prioritizes filename-based matches for more intuitive results
-- Reduces clutter from hidden, cache, and build directories
-- Keeps default suggestions focused on visible project files
+- When reading or editing a file, discovers `AGENTS.md` / `CLAUDE.md` in that file's directory and ancestor directories
+- Newly discovered guidance is injected into tool results, scoped to the current context
 
 ### 6. LSP Tool Suite
 
@@ -85,12 +78,39 @@ A cleaned-up, minimal LSP toolset. The extension keeps only the two LSP tools th
 
 Supported languages: c/cpp, go, java, lua, json, python, ruby, rust, svelte, typescript
 
-### 7. Dynamic Subdirectory `AGENTS.md` / `CLAUDE.md`
+### 7. Built-in MCP Client
 
-When the agent reads or edits a file:
+Zero-config MCP client with built-in servers:
 
-- discovers `AGENTS.md` / `CLAUDE.md` in the file's directory and ancestor directories
-- injects newly discovered guidance into tool results
+| Server | Tool Prefix | Source |
+| --- | --- | --- |
+| Context7 | `context7_*` | `https://mcp.context7.com/mcp` |
+| Exa | `exa_*` | `https://mcp.exa.ai/mcp` |
+
+**Custom servers** in `.pi/agent/mcp.json` (project) or `~/.pi/agent/decorated-pi.json` (global). Project overrides global.
+Tool prompts and schemas are cached locally so MCP tools are available immediately on startup, even before servers connect.
+
+```json
+{
+  "mcpServers": {
+    "my-server": {
+      "url": "https://my-mcp.example.com/mcp",
+      "enabled": true
+    },
+    "my-sse": {
+      "url": "https://my-mcp.example.com/sse",
+      "enabled": false
+    },
+    "my-stdio": {
+      "command": "npx",
+      "args": ["-y", "my-mcp-server"],
+      "env": { "DEBUG": "1" }
+    }
+  }
+}
+```
+
+Use `/mcp` to view connection status and registered tools.
 
 ### 8. Extend Providers
 
@@ -102,6 +122,11 @@ Extend providers are registered via `/login` → "Use a subscription":
 | Baidu Qianfan | `qianfan.baidubce.com/v2/coding` |
 | ARK Coding | `ark.cn-beijing.volces.com/api/coding/v3` |
 
+### 9. Other
+
+- **RTK** — integrates [RTK](https://github.com/rtk-ai/rtk) for token-efficient command output.
+- **WakaTime** — tracks coding activity via [WakaTime](https://wakatime.com).
+
 ## Configuration
 
 Runtime settings are stored in:
@@ -112,26 +137,18 @@ Runtime settings are stored in:
 
 ### Module Loading
 
-Modules can be toggled on/off. Changes take effect after `/reload`.
-
-| Module | Default | Effect when disabled |
-| -------- | --------- | --------------------- |
-| `patch` | `true` | Reverts to Pi's built-in `edit` / `write` tools |
-| `safety` | `true` | No secret redaction on `read` / `bash` output |
-| `lsp` | `true` | All `lsp_*` tools unregistered — no diagnostics, hover, etc. |
-| `smart-at` | `true` | Fallback to Pi's built-in `@` file completion |
-| `mcp` | `true` | All `{server}_*` MCP tools unregistered |
-
-Use `/dp-settings` to toggle, or edit the config file directly:
+Modules can be toggled on/off by `/dp-settings`. Changes take effect after `/reload`.
 
 ```json
 {
   "modules": {
     "patch": true,
     "safety": true,
-    "lsp": false,
+    "rtk": true,
+    "lsp": true,
     "smart-at": true,
-    "mcp": true
+    "mcp": true,
+    "wakatime": true
   }
 }
 ```

package/extensions/index.ts

@@ -3,19 +3,138 @@
  */
 
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+
+// Extend prompt cache TTL for all providers: Anthropic 1h, OpenAI 24h
+process.env.PI_CACHE_RETENTION ??= "long";
 import { setupSafety } from "./safety/index.js";
 import { setupModelIntegration } from "./model-integration";
 import { setupSlash } from "./slash";
 import { setupSubdirAgents } from "./subdir-agents";
 import { setupSessionTitle } from "./session-title";
 import { setupIO } from "./io";
-import { setupGuidance } from "./guidance";
 import { setupLsp } from "./lsp/index";
+import { collectLspDependencyStatuses } from "./lsp/servers";
 import { setupProviders } from "./providers/index";
-import { setupSmartAt } from "./smart-at";
+import { getSmartAtDependencyStatuses, setupSmartAt } from "./smart-at";
 import { setupMcp } from "./mcp/index.js";
+import { collectMcpDependencyStatuses } from "./mcp/builtin";
+import { setupWakatime } from "./wakatime";
+import { findSystemRtk, getRtkDependencyStatuses, setupRtkIntegration, type DependencyStatus } from "./rtk";
 import { isModuleEnabled } from "./settings";
 
+function collectDependencyStatuses(cwd: string): DependencyStatus[] {
+  const statuses: DependencyStatus[] = [];
+  if (isModuleEnabled("rtk")) statuses.push(...getRtkDependencyStatuses());
+  if (isModuleEnabled("smart-at")) statuses.push(...getSmartAtDependencyStatuses(cwd));
+  if (isModuleEnabled("lsp")) statuses.push(...collectLspDependencyStatuses(cwd));
+  if (isModuleEnabled("mcp")) statuses.push(...collectMcpDependencyStatuses(cwd));
+  return statuses;
+}
+
+function formatDependencyLines(statuses: DependencyStatus[]): string[] {
+  const missing = statuses.filter((item) => item.state === "missing");
+  const grouped = new Map<string, string[]>();
+
+  for (const item of missing) {
+    const labels = grouped.get(item.module) ?? [];
+    labels.push(item.label);
+    grouped.set(item.module, labels);
+  }
+
+  const lines = ["[decorated-pi] missing dependencies:"];
+  for (const [module, labels] of grouped) {
+    lines.push(`  [${module}] ${labels.join(", ")}`);
+  }
+  return lines;
+}
+
+function setupDependencyReminders(pi: ExtensionAPI) {
+  let notifyTimer: ReturnType<typeof setTimeout> | undefined;
+
+  pi.on("session_start", async (event, ctx) => {
+    if (!ctx.hasUI) return;
+    if (event.reason !== "startup" && event.reason !== "reload") return;
+
+    const statuses = collectDependencyStatuses(ctx.cwd);
+    const missing = statuses.filter((item) => item.state === "missing");
+    if (missing.length === 0) return;
+
+    if (notifyTimer) clearTimeout(notifyTimer);
+    const message = formatDependencyLines(statuses).join("\n");
+
+    // Defer until after pi finishes startup/reload UI rebuild, otherwise
+    // notify() is appended to the chat and then wiped by rebuildChatFromMessages().
+    notifyTimer = setTimeout(() => {
+      notifyTimer = undefined;
+      try {
+        ctx.ui.notify(message, "info");
+      } catch {
+        // Extension context may be stale if another reload/session switch happened.
+      }
+    }, 0);
+  });
+
+  pi.on("session_shutdown", async () => {
+    if (!notifyTimer) return;
+    clearTimeout(notifyTimer);
+    notifyTimer = undefined;
+  });
+}
+
+const DECORATED_PI_GUIDANCE_MARKER = "## Decorated Pi Guidance";
+
+function setupGuidance(pi: ExtensionAPI) {
+  pi.on("before_agent_start", async (event) => {
+    // Remove "Current date: YYYY-MM-DD" from system prompt to improve cache stability
+    let prompt = event.systemPrompt.replace(/\nCurrent date: \d{4}-\d{2}-\d{2}/, "");
+
+    if (!prompt.includes(DECORATED_PI_GUIDANCE_MARKER)) {
+      const guidance = [
+        DECORATED_PI_GUIDANCE_MARKER,
+        "",
+        "- Before acting on a user's prompt, ensure you fully understand their needs. If the intent is ambiguous, ask clarifying questions. Proceed only when the intent is clear.",
+        "- Look before you leap! Ensure you have conducted thorough research before taking any action.",
+        "- Exercise caution when performing any **write** operations, especially when you are in a research or exploration phase.",
+        "- You don't need to read **AGENTS.md** or **CLAUDE.md** files unless you're explicitly asked to, these files will loaded automatically if neccessary.",
+        "- CAUTION: Do not perform write operations in the following directories unless explicitly instructed: `node_modules`, `venv`, `env`, `__pycache__`, `.git` or any other hidden directories.",
+        "",
+        "### Secret Redaction",
+        "",
+        "- When you see masked secret values (e.g. `sk-***...***` where `*`, `#`, or `?` are mask characters), the real value has been redacted by the system. Do not attempt to read or guess it. If you need the secret, use tools like `jq` or `grep` to extract it from the original source file.",
+      ].join("\n");
+
+      prompt = `${prompt}\n\n${guidance}`;
+    }
+
+    sortSystemPromptOptions(event.systemPromptOptions);
+    return { systemPrompt: prompt };
+  });
+}
+
+/** Sort all fields in systemPromptOptions alphabetically for stable system prompt. */
+export function sortSystemPromptOptions(opts: {
+  toolSnippets?: Record<string, string>;
+  selectedTools?: string[];
+  promptGuidelines?: string[];
+  skills?: Array<{ name: string; description: string; filePath: string }>;
+}) {
+  const sortedToolNames = Object.keys(opts.toolSnippets ?? {}).sort((a, b) => a.localeCompare(b));
+  const sortedToolSnippets: Record<string, string> = {};
+  for (const name of sortedToolNames) {
+    sortedToolSnippets[name] = opts.toolSnippets![name];
+  }
+  opts.toolSnippets = sortedToolSnippets;
+  if (opts.selectedTools) {
+    opts.selectedTools = sortedToolNames;
+  }
+  if (opts.promptGuidelines) {
+    opts.promptGuidelines = [...opts.promptGuidelines].sort((a, b) => a.localeCompare(b));
+  }
+  if (opts.skills) {
+    opts.skills = [...opts.skills].sort((a, b) => a.name.localeCompare(b.name));
+  }
+}
+
 export default function (pi: ExtensionAPI) {
   // Always loaded — core commands and providers
   setupSlash(pi);
@@ -24,11 +143,14 @@ export default function (pi: ExtensionAPI) {
   setupSubdirAgents(pi);
   setupSessionTitle(pi);
   setupGuidance(pi);
+  setupDependencyReminders(pi);
 
   // Configurable modules
-  if (isModuleEnabled("patch"))     setupIO(pi);
-  if (isModuleEnabled("safety"))    setupSafety(pi);
-  if (isModuleEnabled("lsp"))       setupLsp(pi);
-  if (isModuleEnabled("smart-at"))  setupSmartAt(pi);
-  if (isModuleEnabled("mcp"))       setupMcp(pi);
-}
+  if (isModuleEnabled("patch")) setupIO(pi);
+  if (isModuleEnabled("safety")) setupSafety(pi);
+  if (isModuleEnabled("lsp")) setupLsp(pi);
+  if (isModuleEnabled("smart-at")) setupSmartAt(pi);
+  if (isModuleEnabled("mcp")) setupMcp(pi);
+  if (isModuleEnabled("wakatime")) setupWakatime(pi);
+  if (isModuleEnabled("rtk") && findSystemRtk()) setupRtkIntegration(pi);
+}

package/extensions/io.ts

@@ -193,7 +193,7 @@ function normalizeDisplayText(text: string): string {
 }
 
 function replaceTabs(text: string): string {
-  return text.replace(/\t/g, "   ");
+  return text.replace(/\t/g, "    ");
 }
 
 function highlightSingleLine(line: string, lang: string | undefined): string {
@@ -302,7 +302,7 @@ function appendPatchDiffChildren(parent: Box, body: string, theme: any): void {
 
   const flush = () => {
     if (buffer.length === 0) return;
-    parent.addChild(new Text(renderDiff(buffer.join("\n")), 0, 0));
+    parent.addChild(new Text(renderDiff(replaceTabs(buffer.join("\n"))), 0, 0));
     buffer = [];
   };
 
@@ -336,7 +336,7 @@ export function buildPatchCallComponent(component: PatchCallComponent, args: any
   if (args?.path) {
     label = theme.fg("accent", args.path);
     if (args.overwrite) {
-      label += theme.fg("warning", " [overwrite]");
+      label += theme.fg("error", " [overwrite]");
     } else if (args.edits?.length > 0) {
       label += theme.fg("dim", ` (${args.edits.length} edit${args.edits.length > 1 ? "s" : ""})`);
     }
@@ -447,6 +447,8 @@ export function setupIO(pi: ExtensionAPI) {
     promptGuidelines: [
       "Always prefer modifying files with PATCH tool over bash commands or python scripts.",
       "For full-file replacement, always use patch tool to prevent unintended edits or data loss.",
+      "To prevent hallucinations: 1. Keep each edit batch ≤ 5 changes; 2. Process remaining revisions in sequential steps",
+      "On repeated failures: read the file first to confirm information accuracy.",
     ],
     parameters: PatchSchema,
     renderShell: "self",

package/extensions/lsp/servers.ts

@@ -1,8 +1,10 @@
 /**
  * LSP Server Config — language detection, server commands, workspace roots.
  */
-import { existsSync } from "node:fs";
+import { existsSync, readdirSync, type Dirent } from "node:fs";
+import { spawnSync } from "node:child_process";
 import { dirname, extname, isAbsolute, join, resolve } from "node:path";
+import type { DependencyStatus } from "../rtk";
 
 // ─── File extension → language mapping ────────────────────────────────────
 
@@ -39,8 +41,8 @@ const LANGUAGE_SERVERS: Record<string, Omit<LanguageConfig, "is_project_local">>
     install_hint: "Install clangd and ensure the clangd binary is on PATH.",
   },
   python: {
-    language: "python", command: "pylsp", args: [],
-    install_hint: "Install Python LSP with: pip install python-lsp-server",
+    language: "python", command: "pyright-langserver", args: ["--stdio"],
+    install_hint: "Install a Python LSP and ensure the pyright-langserver binary is on PATH.",
   },
   rust: {
     language: "rust", command: "rust-analyzer", args: [],
@@ -110,6 +112,54 @@ export function languageIdForFile(filePath: string): string | undefined {
   return detectLanguage(filePath);
 }
 
+export function detectProjectLanguages(cwd: string, limit = 1500): Set<string> {
+  const found = new Set<string>();
+  let seen = 0;
+  const skipDirs = new Set([".git", "node_modules", ".pi", "dist", "build", "coverage"]);
+
+  function walk(dir: string): void {
+    if (seen >= limit) return;
+    let entries: Dirent[] = [];
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      if (seen >= limit) return;
+      const full = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (skipDirs.has(entry.name) || entry.name.startsWith('.')) continue;
+        walk(full);
+      } else if (entry.isFile()) {
+        seen++;
+        const language = detectLanguage(full);
+        if (language) found.add(language);
+      }
+    }
+  }
+
+  walk(cwd);
+  return found;
+}
+
+export function collectLspDependencyStatuses(cwd: string): DependencyStatus[] {
+  const statuses: DependencyStatus[] = [];
+  const seen = new Set<string>();
+  for (const language of listSupportedLanguages()) {
+    const cfg = getServerConfig(language, cwd);
+    if (!cfg || seen.has(cfg.command)) continue;
+    seen.add(cfg.command);
+    statuses.push({
+      module: "lsp",
+      label: cfg.command,
+      state: commandExists(cfg.command) ? "ok" : "missing",
+      detail: cfg.install_hint,
+    });
+  }
+  return statuses;
+}
+
 export function findWorkspaceRoot(
   filePath: string,
   fallback = process.cwd(),
@@ -124,6 +174,16 @@ export function findWorkspaceRoot(
 
 // ─── Internal helpers ─────────────────────────────────────────────────────
 
+function commandExists(command: string): boolean {
+  if (isAbsolute(command) || command.includes("/") || command.includes("\\")) {
+    return existsSync(command);
+  }
+  const result = process.platform === "win32"
+    ? spawnSync("where", [command], { encoding: "utf-8" })
+    : spawnSync(process.env.SHELL || "sh", ["-lc", `command -v '${command.replace(/'/g, `'"'"'`)}'`], { encoding: "utf-8" });
+  return result.status === 0;
+}
+
 function resolveLocalBinary(
   command: string,
   cwd: string,