context-mode 0.9.15 → 0.9.17

package/.claude-plugin/marketplace.json

@@ -13,7 +13,7 @@
       "name": "context-mode",
       "source": "./",
       "description": "Claude Code MCP plugin that saves 98% of your context window. Sandboxed code execution in 11 languages, FTS5 knowledge base with BM25 ranking, and intent-driven search.",
-      "version": "0.9.15",
+      "version": "0.9.17",
       "author": {
         "name": "Mert Koseoğlu"
       },

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "context-mode",
-  "version": "0.9.15",
+  "version": "0.9.17",
   "description": "Claude Code MCP plugin that saves 98% of your context window. Sandboxed code execution in 11 languages, FTS5 knowledge base with BM25 ranking, and intent-driven search.",
   "author": {
     "name": "Mert Koseoğlu",

package/README.md

@@ -190,7 +190,9 @@ with code examples. Then run /context-mode:stats.
 - **Claude Code** with MCP support
 - Optional: Bun (auto-detected, 3-5x faster JS/TS)
 
-## Development
+## Contributing
+
+We welcome contributions! See [CONTRIBUTING.md](CONTRIBUTING.md) for the full local development workflow, TDD guidelines, and how to test your changes in a live Claude Code session.
 
 ```bash
 git clone https://github.com/mksglu/claude-context-mode.git

package/build/executor.js

@@ -36,7 +36,14 @@ export class PolyglotExecutor {
         const tmpDir = mkdtempSync(join(tmpdir(), "ctx-mode-"));
         try {
             const filePath = this.#writeScript(tmpDir, code, language);
-            const cmd = buildCommand(this.#runtimes, language, filePath);
+            let cmd;
+            try {
+                cmd = buildCommand(this.#runtimes, language, filePath);
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                return { exitCode: 1, stdout: "", stderr: msg, timedOut: false };
+            }
             // Rust: compile then run
             if (cmd[0] === "__rust_compile_run__") {
                 return await this.#compileAndRun(filePath, tmpDir, timeout);
@@ -44,7 +51,13 @@ export class PolyglotExecutor {
             return await this.#spawn(cmd, tmpDir, timeout);
         }
         finally {
-            rmSync(tmpDir, { recursive: true, force: true });
+            try {
+                rmSync(tmpDir, { recursive: true, force: true });
+            }
+            catch {
+                // On Windows, EBUSY/EPERM is common due to delayed handle release
+                // after child process exit. Silently ignore — OS cleans temp dirs.
+            }
         }
     }
     async executeFile(opts) {

package/build/runtime.d.ts

@@ -9,7 +9,7 @@ export interface RuntimeMap {
     javascript: string;
     typescript: string | null;
     python: string | null;
-    shell: string;
+    shell: string | null;
     ruby: string | null;
     go: string | null;
     rust: string | null;

package/build/runtime.js

@@ -40,7 +40,15 @@ export function detectRuntimes() {
             : commandExists("python")
                 ? "python"
                 : null,
-        shell: commandExists("bash") ? "bash" : "sh",
+        shell: commandExists("bash")
+            ? "bash"
+            : commandExists("sh")
+                ? "sh"
+                : commandExists("powershell")
+                    ? "powershell"
+                    : commandExists("cmd.exe")
+                        ? "cmd.exe"
+                        : null,
         ruby: commandExists("ruby") ? "ruby" : null,
         go: commandExists("go") ? "go" : null,
         rust: commandExists("rustc") ? "rustc" : null,
@@ -73,7 +81,12 @@ export function getRuntimeSummary(runtimes) {
     else {
         lines.push(`  Python:     not available`);
     }
-    lines.push(`  Shell:      ${runtimes.shell} (${getVersion(runtimes.shell)})`);
+    if (runtimes.shell) {
+        lines.push(`  Shell:      ${runtimes.shell} (${getVersion(runtimes.shell)})`);
+    }
+    else {
+        lines.push(`  Shell:      not available`);
+    }
     // Optional runtimes — only show if available
     if (runtimes.ruby)
         lines.push(`  Ruby:       ${runtimes.ruby} (${getVersion(runtimes.ruby)})`);
@@ -96,7 +109,9 @@ export function getRuntimeSummary(runtimes) {
     return lines.join("\n");
 }
 export function getAvailableLanguages(runtimes) {
-    const langs = ["javascript", "shell"];
+    const langs = ["javascript"];
+    if (runtimes.shell)
+        langs.push("shell");
     if (runtimes.typescript)
         langs.push("typescript");
     if (runtimes.python)
@@ -138,6 +153,9 @@ export function buildCommand(runtimes, language, filePath) {
             }
             return [runtimes.python, filePath];
         case "shell":
+            if (!runtimes.shell) {
+                throw new Error("No shell runtime available. Install bash, sh, powershell, or cmd.");
+            }
             return [runtimes.shell, filePath];
         case "ruby":
             if (!runtimes.ruby) {

package/build/server.d.ts

@@ -1,2 +1,8 @@
 #!/usr/bin/env node
-export {};
+/**
+ * Parse FTS5 highlight markers to find match positions in the
+ * original (marker-free) text. Returns character offsets into the
+ * stripped content where each matched token begins.
+ */
+export declare function positionsFromHighlight(highlighted: string): number[];
+export declare function extractSnippet(content: string, query: string, maxLen?: number, highlighted?: string): string;

package/build/server.js

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { createRequire } from "node:module";
 import { z } from "zod";
 import { PolyglotExecutor } from "./executor.js";
 import { ContentStore, cleanupStaleDBs } from "./store.js";
@@ -51,25 +52,76 @@ const bunNote = hasBunRuntime()
 // ─────────────────────────────────────────────────────────
 // Helper: smart snippet extraction — returns windows around
 // matching query terms instead of dumb truncation
+//
+// When `highlighted` is provided (from FTS5 `highlight()` with
+// STX/ETX markers), match positions are derived from the markers.
+// This is the authoritative source — FTS5 uses the exact same
+// tokenizer that produced the BM25 match, so stemmed variants
+// like "configuration" matching query "configure" are found
+// correctly. Falls back to indexOf on raw terms when highlighted
+// is absent (non-FTS codepath).
 // ─────────────────────────────────────────────────────────
-function extractSnippet(content, query, maxLen = 1500) {
+const STX = "\x02";
+const ETX = "\x03";
+/**
+ * Parse FTS5 highlight markers to find match positions in the
+ * original (marker-free) text. Returns character offsets into the
+ * stripped content where each matched token begins.
+ */
+export function positionsFromHighlight(highlighted) {
+    const positions = [];
+    let cleanOffset = 0;
+    let i = 0;
+    while (i < highlighted.length) {
+        if (highlighted[i] === STX) {
+            // Record position of this match in the clean text
+            positions.push(cleanOffset);
+            i++; // skip STX
+            // Advance through matched text until ETX
+            while (i < highlighted.length && highlighted[i] !== ETX) {
+                cleanOffset++;
+                i++;
+            }
+            if (i < highlighted.length)
+                i++; // skip ETX
+        }
+        else {
+            cleanOffset++;
+            i++;
+        }
+    }
+    return positions;
+}
+/** Strip STX/ETX markers to recover original content. */
+function stripMarkers(highlighted) {
+    return highlighted.replaceAll(STX, "").replaceAll(ETX, "");
+}
+export function extractSnippet(content, query, maxLen = 1500, highlighted) {
     if (content.length <= maxLen)
         return content;
-    const terms = query
-        .toLowerCase()
-        .split(/\s+/)
-        .filter((t) => t.length > 2);
-    const lower = content.toLowerCase();
-    // Find all positions where query terms appear
+    // Derive match positions from FTS5 highlight markers when available
     const positions = [];
-    for (const term of terms) {
-        let idx = lower.indexOf(term);
-        while (idx !== -1) {
-            positions.push(idx);
-            idx = lower.indexOf(term, idx + 1);
+    if (highlighted) {
+        for (const pos of positionsFromHighlight(highlighted)) {
+            positions.push(pos);
         }
     }
-    // No term matches — return start (BM25 matched on stems/variants)
+    // Fallback: indexOf on raw query terms (non-FTS codepath)
+    if (positions.length === 0) {
+        const terms = query
+            .toLowerCase()
+            .split(/\s+/)
+            .filter((t) => t.length > 2);
+        const lower = content.toLowerCase();
+        for (const term of terms) {
+            let idx = lower.indexOf(term);
+            while (idx !== -1) {
+                positions.push(idx);
+                idx = lower.indexOf(term, idx + 1);
+            }
+        }
+    }
+    // No matches at all — return prefix
     if (positions.length === 0) {
         return content.slice(0, maxLen) + "\n…";
     }
@@ -541,7 +593,7 @@ server.registerTool("search", {
                 .map((r, i) => {
                 const header = `--- [${r.source}] ---`;
                 const heading = `### ${r.title}`;
-                const snippet = extractSnippet(r.content, q);
+                const snippet = extractSnippet(r.content, q, 1500, r.highlighted);
                 return `${header}\n${heading}\n\n${snippet}`;
             })
                 .join("\n\n");
@@ -577,75 +629,48 @@ server.registerTool("search", {
     }
 });
 // ─────────────────────────────────────────────────────────
+// Turndown path resolution (external dep, like better-sqlite3)
+// ─────────────────────────────────────────────────────────
+let _turndownPath = null;
+let _gfmPluginPath = null;
+function resolveTurndownPath() {
+    if (!_turndownPath) {
+        const require = createRequire(import.meta.url);
+        _turndownPath = require.resolve("turndown");
+    }
+    return _turndownPath;
+}
+function resolveGfmPluginPath() {
+    if (!_gfmPluginPath) {
+        const require = createRequire(import.meta.url);
+        _gfmPluginPath = require.resolve("turndown-plugin-gfm");
+    }
+    return _gfmPluginPath;
+}
+// ─────────────────────────────────────────────────────────
 // Tool: fetch_and_index
 // ─────────────────────────────────────────────────────────
-const HTML_TO_MARKDOWN_CODE = `
-const url = process.argv[1];
-if (!url) { console.error("No URL provided"); process.exit(1); }
+function buildFetchCode(url) {
+    const turndownPath = JSON.stringify(resolveTurndownPath());
+    const gfmPath = JSON.stringify(resolveGfmPluginPath());
+    return `
+const TurndownService = require(${turndownPath});
+const { gfm } = require(${gfmPath});
+const url = ${JSON.stringify(url)};
 
 async function main() {
   const resp = await fetch(url);
   if (!resp.ok) { console.error("HTTP " + resp.status); process.exit(1); }
+  const html = await resp.text();
 
-  let html = await resp.text();
-
-  // Strip script, style, nav, header, footer tags with content
-  html = html.replace(/<script[^>]*>[\\s\\S]*?<\\/script>/gi, "");
-  html = html.replace(/<style[^>]*>[\\s\\S]*?<\\/style>/gi, "");
-  html = html.replace(/<nav[^>]*>[\\s\\S]*?<\\/nav>/gi, "");
-  html = html.replace(/<header[^>]*>[\\s\\S]*?<\\/header>/gi, "");
-  html = html.replace(/<footer[^>]*>[\\s\\S]*?<\\/footer>/gi, "");
-
-  // Convert headings to markdown
-  html = html.replace(/<h1[^>]*>(.*?)<\\/h1>/gi, "\\n# $1\\n");
-  html = html.replace(/<h2[^>]*>(.*?)<\\/h2>/gi, "\\n## $1\\n");
-  html = html.replace(/<h3[^>]*>(.*?)<\\/h3>/gi, "\\n### $1\\n");
-  html = html.replace(/<h4[^>]*>(.*?)<\\/h4>/gi, "\\n#### $1\\n");
-
-  // Convert code blocks
-  html = html.replace(/<pre[^>]*><code[^>]*class="[^"]*language-(\\w+)"[^>]*>([\\s\\S]*?)<\\/code><\\/pre>/gi,
-    (_, lang, code) => "\\n\\\`\\\`\\\`" + lang + "\\n" + decodeEntities(code) + "\\n\\\`\\\`\\\`\\n");
-  html = html.replace(/<pre[^>]*><code[^>]*>([\\s\\S]*?)<\\/code><\\/pre>/gi,
-    (_, code) => "\\n\\\`\\\`\\\`\\n" + decodeEntities(code) + "\\n\\\`\\\`\\\`\\n");
-  html = html.replace(/<code[^>]*>([^<]*)<\\/code>/gi, "\\\`$1\\\`");
-
-  // Convert links
-  html = html.replace(/<a[^>]*href="([^"]*)"[^>]*>(.*?)<\\/a>/gi, "[$2]($1)");
-
-  // Convert lists
-  html = html.replace(/<li[^>]*>(.*?)<\\/li>/gi, "- $1\\n");
-
-  // Convert paragraphs and line breaks
-  html = html.replace(/<p[^>]*>(.*?)<\\/p>/gi, "\\n$1\\n");
-  html = html.replace(/<br\\s*\\/?>/gi, "\\n");
-  html = html.replace(/<hr\\s*\\/?>/gi, "\\n---\\n");
-
-  // Strip remaining HTML tags
-  html = html.replace(/<[^>]+>/g, "");
-
-  // Decode HTML entities
-  html = decodeEntities(html);
-
-  // Clean up whitespace
-  html = html.replace(/\\n{3,}/g, "\\n\\n").trim();
-
-  console.log(html);
+  const td = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+  td.use(gfm);
+  td.remove(['script', 'style', 'nav', 'header', 'footer', 'noscript']);
+  console.log(td.turndown(html));
 }
-
-function decodeEntities(s) {
-  return s
-    .replace(/&amp;/g, "&")
-    .replace(/&lt;/g, "<")
-    .replace(/&gt;/g, ">")
-    .replace(/&quot;/g, '"')
-    .replace(/&#39;/g, "'")
-    .replace(/&#x27;/g, "'")
-    .replace(/&#x2F;/g, "/")
-    .replace(/&nbsp;/g, " ");
-}
-
 main();
 `;
+}
 server.registerTool("fetch_and_index", {
     title: "Fetch & Index URL",
     description: "Fetches URL content, converts HTML to markdown, indexes into searchable knowledge base, " +
@@ -661,7 +686,7 @@ server.registerTool("fetch_and_index", {
 }, async ({ url, source }) => {
     try {
         // Execute fetch inside subprocess — raw HTML never enters context
-        const fetchCode = `process.argv[1] = ${JSON.stringify(url)};\n${HTML_TO_MARKDOWN_CODE}`;
+        const fetchCode = buildFetchCode(url);
         const result = await executor.execute({
             language: "javascript",
             code: fetchCode,
@@ -822,7 +847,7 @@ server.registerTool("batch_execute", {
             queryResults.push("");
             if (results.length > 0) {
                 for (const r of results) {
-                    const snippet = extractSnippet(r.content, query);
+                    const snippet = extractSnippet(r.content, query, 1500, r.highlighted);
                     queryResults.push(`### ${r.title}`);
                     queryResults.push(snippet);
                     queryResults.push("");

package/build/store.d.ts

@@ -20,6 +20,7 @@ export interface SearchResult {
     rank: number;
     contentType: "code" | "prose";
     matchLayer?: "porter" | "trigram" | "fuzzy";
+    highlighted?: string;
 }
 export interface StoreStats {
     sources: number;

package/build/store.js

@@ -271,7 +271,8 @@ export class ContentStore {
         chunks.content,
         chunks.content_type,
         sources.label,
-        bm25(chunks, 2.0, 1.0) AS rank
+        bm25(chunks, 2.0, 1.0) AS rank,
+        highlight(chunks, 1, char(2), char(3)) AS highlighted
       FROM chunks
       JOIN sources ON sources.id = chunks.source_id
       WHERE chunks MATCH ? ${sourceFilter}
@@ -288,6 +289,7 @@ export class ContentStore {
             source: r.label,
             rank: r.rank,
             contentType: r.content_type,
+            highlighted: r.highlighted,
         }));
     }
     // ── Trigram Search (Layer 2) ──
@@ -302,7 +304,8 @@ export class ContentStore {
         chunks_trigram.content,
         chunks_trigram.content_type,
         sources.label,
-        bm25(chunks_trigram, 2.0, 1.0) AS rank
+        bm25(chunks_trigram, 2.0, 1.0) AS rank,
+        highlight(chunks_trigram, 1, char(2), char(3)) AS highlighted
       FROM chunks_trigram
       JOIN sources ON sources.id = chunks_trigram.source_id
       WHERE chunks_trigram MATCH ? ${sourceFilter}
@@ -319,6 +322,7 @@ export class ContentStore {
             source: r.label,
             rank: r.rank,
             contentType: r.content_type,
+            highlighted: r.highlighted,
         }));
     }
     // ── Fuzzy Correction (Layer 3) ──

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "context-mode",
-  "version": "0.9.15",
+  "version": "0.9.17",
   "type": "module",
   "description": "Claude Code MCP plugin that saves 98% of your context window. Sandboxed code execution, FTS5 knowledge base, and intent-driven search.",
   "author": "Mert Koseoğlu",
@@ -38,7 +38,7 @@
   ],
   "scripts": {
     "build": "tsc",
-    "bundle": "esbuild src/server.ts --bundle --platform=node --target=node18 --format=esm --outfile=server.bundle.mjs --external:better-sqlite3 --minify",
+    "bundle": "esbuild src/server.ts --bundle --platform=node --target=node18 --format=esm --outfile=server.bundle.mjs --external:better-sqlite3 --external:turndown --external:turndown-plugin-gfm --external:@mixmark-io/domino --minify",
     "prepublishOnly": "npm run build",
     "dev": "npx tsx src/server.ts",
     "setup": "npx tsx src/cli.ts setup",
@@ -56,18 +56,23 @@
     "test:stream-cap": "npx tsx tests/stream-cap.test.ts",
     "test:search-wiring": "npx tsx tests/search-wiring.test.ts",
     "test:search-fallback": "npx tsx tests/search-fallback-integration.test.ts",
+    "test:turndown": "npx tsx tests/turndown.test.ts",
     "test:all": "for f in tests/*.test.ts; do npx tsx \"$f\" || exit 1; done"
   },
   "dependencies": {
     "@clack/prompts": "^1.0.1",
+    "@mixmark-io/domino": "^2.2.0",
     "@modelcontextprotocol/sdk": "^1.26.0",
     "better-sqlite3": "^12.6.2",
     "picocolors": "^1.1.1",
+    "turndown": "^7.2.0",
+    "turndown-plugin-gfm": "^1.0.2",
     "zod": "^3.25.0"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.13",
     "@types/node": "^22.19.11",
+    "@types/turndown": "^5.0.5",
     "esbuild": "^0.27.3",
     "tsx": "^4.21.0",
     "typescript": "^5.7.0"