mcp-local-rag 0.4.2 → 0.5.1

package/README.md

@@ -2,6 +2,8 @@
 
 [![npm version](https://img.shields.io/npm/v/mcp-local-rag.svg)](https://www.npmjs.com/package/mcp-local-rag)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![TypeScript](https://img.shields.io/badge/TypeScript-5.0-blue.svg?logo=typescript&logoColor=white)](https://www.typescriptlang.org/)
+[![MCP Registry](https://img.shields.io/badge/MCP-Registry-green.svg)](https://registry.modelcontextprotocol.io/)
 
 Local RAG for developers using MCP.
 Semantic search with keyword boost for exact technical terms — fully private, zero setup.
@@ -86,8 +88,8 @@ You want AI to search your documents—technical specs, research papers, interna
 
 ## Usage
 
-The server provides 5 MCP tools: ingest, search, list, delete, status
-(`ingest_file`, `query_documents`, `list_files`, `delete_file`, `status`).
+The server provides 6 MCP tools: ingest file, ingest data, search, list, delete, status
+(`ingest_file`, `ingest_data`, `query_documents`, `list_files`, `delete_file`, `status`).
 
 ### Ingesting Documents
 
@@ -99,6 +101,23 @@ Supports PDF, DOCX, TXT, and Markdown. The server extracts text, splits it into
 
 Re-ingesting the same file replaces the old version automatically.
 
+### Ingesting HTML Content
+
+Use `ingest_data` to ingest HTML content retrieved by your AI assistant (via web fetch, curl, browser tools, etc.):
+
+```
+"Fetch https://example.com/docs and ingest the HTML"
+```
+
+The server extracts main content using Readability (removes navigation, ads, etc.), converts to Markdown, and indexes it. Perfect for:
+- Web documentation
+- HTML retrieved by the AI assistant
+- Clipboard content
+
+HTML is automatically cleaned—you get the article content, not the boilerplate.
+
+> **Note:** The RAG server itself doesn't fetch web content—your AI assistant retrieves it and passes the HTML to `ingest_data`. This keeps the server fully local while letting you index any content your assistant can access. Please respect website terms of service and copyright when ingesting external content.
+
 ### Searching Documents
 
 ```
@@ -169,6 +188,42 @@ When you search:
 
 The keyword boost ensures exact terms like `useEffect` or error codes rank higher when they match.
 
+## Agent Skills
+
+[Agent Skills](https://agentskills.io/) provide optimized prompts that help AI assistants use RAG tools more effectively. Install skills for better query formulation, result interpretation, and ingestion workflows:
+
+```bash
+# Claude Code (project-level)
+npx mcp-local-rag-skills --claude-code
+
+# Claude Code (user-level)
+npx mcp-local-rag-skills --claude-code --global
+
+# Codex
+npx mcp-local-rag-skills --codex
+```
+
+Skills include:
+- **Query optimization**: Better search query formulation
+- **Result interpretation**: Score thresholds and filtering guidelines
+- **HTML ingestion**: Format selection and source naming
+
+### Ensuring Skill Activation
+
+Skills are loaded automatically in most cases—AI assistants scan skill metadata and load relevant instructions when needed. For consistent behavior:
+
+**Option 1: Explicit request (natural language)**
+Before RAG operations, request in natural language:
+- "Use the mcp-local-rag skill for this search"
+- "Apply RAG best practices from skills"
+
+**Option 2: Add to agent instruction file**
+Add to your `AGENTS.md`, `CLAUDE.md`, or other agent instruction file:
+```
+When using query_documents, ingest_file, or ingest_data tools,
+apply the mcp-local-rag skill for optimal query formulation and result interpretation.
+```
+
 <details>
 <summary><strong>Configuration</strong></summary>
 
@@ -301,7 +356,7 @@ Yes, after the first model download (~90MB).
 Cloud services offer better accuracy at scale but require sending data externally. This trades some accuracy for complete privacy and zero runtime cost.
 
 **What file formats are supported?**
-PDF, DOCX, TXT, Markdown. Not yet: Excel, PowerPoint, images, HTML.
+PDF, DOCX, TXT, Markdown, and HTML (via `ingest_data`). Not yet: Excel, PowerPoint, images.
 
 **Can I change the embedding model?**
 Yes, but you must delete your database and re-ingest all documents. Different models produce incompatible vector dimensions.

package/dist/bin/install-skills.d.ts

@@ -0,0 +1,17 @@
+#!/usr/bin/env node
+/**
+ * MCP Local RAG Skills Installer
+ *
+ * Installs skills to various AI coding assistants:
+ * - Claude Code (project or global)
+ * - OpenAI Codex
+ * - Custom path
+ *
+ * Usage:
+ *   npx mcp-local-rag-skills --claude-code          # Project-level
+ *   npx mcp-local-rag-skills --claude-code --global # User-level
+ *   npx mcp-local-rag-skills --codex                # Codex
+ *   npx mcp-local-rag-skills --path /custom/path    # Custom
+ */
+export {};
+//# sourceMappingURL=install-skills.d.ts.map

package/dist/bin/install-skills.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"install-skills.d.ts","sourceRoot":"","sources":["../../src/bin/install-skills.ts"],"names":[],"mappings":";AAEA;;;;;;;;;;;;;GAaG"}

package/dist/bin/install-skills.js

@@ -0,0 +1,194 @@
+#!/usr/bin/env node
+"use strict";
+/**
+ * MCP Local RAG Skills Installer
+ *
+ * Installs skills to various AI coding assistants:
+ * - Claude Code (project or global)
+ * - OpenAI Codex
+ * - Custom path
+ *
+ * Usage:
+ *   npx mcp-local-rag-skills --claude-code          # Project-level
+ *   npx mcp-local-rag-skills --claude-code --global # User-level
+ *   npx mcp-local-rag-skills --codex                # Codex
+ *   npx mcp-local-rag-skills --path /custom/path    # Custom
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_fs_1 = require("node:fs");
+const node_os_1 = require("node:os");
+const node_path_1 = require("node:path");
+// ============================================
+// Constants
+// ============================================
+// Skills source directory (relative to dist/bin when compiled)
+// dist/bin/install-skills.js -> dist/skills/mcp-local-rag
+// But skills are actually in package root: skills/mcp-local-rag
+// So from dist/bin, go up twice: ../.. then skills/mcp-local-rag
+const SKILLS_SOURCE = (0, node_path_1.resolve)(__dirname, '..', '..', 'skills', 'mcp-local-rag');
+// Codex home directory (supports CODEX_HOME environment variable)
+// https://developers.openai.com/codex/local-config/
+const CODEX_HOME = process.env['CODEX_HOME'] || (0, node_path_1.join)((0, node_os_1.homedir)(), '.codex');
+// Installation targets
+const TARGETS = {
+    'claude-code-project': './.claude/skills/mcp-local-rag',
+    'claude-code-global': (0, node_path_1.join)((0, node_os_1.homedir)(), '.claude', 'skills', 'mcp-local-rag'),
+    'codex-project': './.codex/skills/mcp-local-rag',
+    'codex-global': (0, node_path_1.join)(CODEX_HOME, 'skills', 'mcp-local-rag'),
+};
+function parseArgs(args) {
+    const options = {
+        target: 'claude-code-project',
+        help: false,
+    };
+    for (let i = 0; i < args.length; i++) {
+        const arg = args[i];
+        switch (arg) {
+            case '--help':
+            case '-h':
+                options.help = true;
+                break;
+            case '--claude-code':
+                // Check for --global flag
+                if (args[i + 1] === '--global') {
+                    options.target = 'claude-code-global';
+                    i++; // Skip next arg
+                }
+                else {
+                    options.target = 'claude-code-project';
+                }
+                break;
+            case '--codex':
+                // Check for --project or --global flag
+                if (args[i + 1] === '--project') {
+                    options.target = 'codex-project';
+                    i++; // Skip next arg
+                }
+                else if (args[i + 1] === '--global') {
+                    options.target = 'codex-global';
+                    i++; // Skip next arg
+                }
+                else {
+                    // Default to global (matches previous behavior)
+                    options.target = 'codex-global';
+                }
+                break;
+            case '--path': {
+                const pathArg = args[i + 1];
+                if (!pathArg) {
+                    console.error('Error: --path requires a path argument');
+                    process.exit(1);
+                }
+                options.target = 'custom';
+                options.customPath = pathArg;
+                i++; // Skip next arg
+                break;
+            }
+            default:
+                if (arg?.startsWith('-')) {
+                    console.error(`Unknown option: ${arg}`);
+                    process.exit(1);
+                }
+        }
+    }
+    return options;
+}
+// ============================================
+// Help Message
+// ============================================
+function printHelp() {
+    console.log(`
+MCP Local RAG Skills Installer
+
+Usage:
+  npx mcp-local-rag-skills [options]
+
+Options:
+  --claude-code          Install to project-level Claude Code skills
+                         (./.claude/skills/)
+
+  --claude-code --global Install to user-level Claude Code skills
+                         (~/.claude/skills/)
+
+  --codex                Install to user-level Codex skills (default)
+                         ($CODEX_HOME/skills/ or ~/.codex/skills/)
+
+  --codex --project      Install to project-level Codex skills
+                         (./.codex/skills/)
+
+  --codex --global       Install to user-level Codex skills
+                         ($CODEX_HOME/skills/ or ~/.codex/skills/)
+
+  --path <path>          Install to custom path
+
+  --help, -h             Show this help message
+
+Examples:
+  npx mcp-local-rag-skills --claude-code
+  npx mcp-local-rag-skills --claude-code --global
+  npx mcp-local-rag-skills --codex
+  npx mcp-local-rag-skills --codex --project
+  npx mcp-local-rag-skills --path ./my-skills/
+`);
+}
+// ============================================
+// Installation
+// ============================================
+function getTargetPath(options) {
+    if (options.target === 'custom') {
+        if (!options.customPath) {
+            console.error('Error: Custom path not specified');
+            process.exit(1);
+        }
+        return (0, node_path_1.resolve)(options.customPath, 'mcp-local-rag');
+    }
+    return TARGETS[options.target];
+}
+function install(targetPath) {
+    // Check source exists
+    if (!(0, node_fs_1.existsSync)(SKILLS_SOURCE)) {
+        console.error(`Error: Skills source not found at ${SKILLS_SOURCE}`);
+        process.exit(1);
+    }
+    // Create target directory
+    const targetDir = (0, node_path_1.dirname)(targetPath);
+    if (!(0, node_fs_1.existsSync)(targetDir)) {
+        (0, node_fs_1.mkdirSync)(targetDir, { recursive: true });
+        console.log(`Created directory: ${targetDir}`);
+    }
+    // Copy skills
+    (0, node_fs_1.cpSync)(SKILLS_SOURCE, targetPath, { recursive: true });
+    console.log(`Installed skills to: ${targetPath}`);
+}
+// ============================================
+// Main
+// ============================================
+function main() {
+    const args = process.argv.slice(2);
+    // Default to help if no args
+    if (args.length === 0) {
+        printHelp();
+        process.exit(0);
+    }
+    const options = parseArgs(args);
+    if (options.help) {
+        printHelp();
+        process.exit(0);
+    }
+    const targetPath = getTargetPath(options);
+    console.log('Installing MCP Local RAG skills...');
+    console.log(`Target: ${options.target}`);
+    console.log(`Path: ${targetPath}`);
+    console.log();
+    install(targetPath);
+    console.log();
+    console.log('Installation complete!');
+    console.log();
+    console.log('The following skills are now available:');
+    console.log('  - mcp-local-rag (SKILL.md)');
+    console.log('  - references/html-ingestion.md');
+    console.log('  - references/query-optimization.md');
+    console.log('  - references/result-refinement.md');
+}
+main();
+//# sourceMappingURL=install-skills.js.map

package/dist/bin/install-skills.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"install-skills.js","sourceRoot":"","sources":["../../src/bin/install-skills.ts"],"names":[],"mappings":";;AAEA;;;;;;;;;;;;;GAaG;;AAEH,qCAAuD;AACvD,qCAAiC;AACjC,yCAAkD;AAElD,+CAA+C;AAC/C,YAAY;AACZ,+CAA+C;AAE/C,+DAA+D;AAC/D,0DAA0D;AAC1D,gEAAgE;AAChE,iEAAiE;AACjE,MAAM,aAAa,GAAG,IAAA,mBAAO,EAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAA;AAE/E,kEAAkE;AAClE,oDAAoD;AACpD,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,IAAA,gBAAI,EAAC,IAAA,iBAAO,GAAE,EAAE,QAAQ,CAAC,CAAA;AAEzE,uBAAuB;AACvB,MAAM,OAAO,GAAG;IACd,qBAAqB,EAAE,gCAAgC;IACvD,oBAAoB,EAAE,IAAA,gBAAI,EAAC,IAAA,iBAAO,GAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,eAAe,CAAC;IAC3E,eAAe,EAAE,+BAA+B;IAChD,cAAc,EAAE,IAAA,gBAAI,EAAC,UAAU,EAAE,QAAQ,EAAE,eAAe,CAAC;CACnD,CAAA;AAYV,SAAS,SAAS,CAAC,IAAc;IAC/B,MAAM,OAAO,GAAY;QACvB,MAAM,EAAE,qBAAqB;QAC7B,IAAI,EAAE,KAAK;KACZ,CAAA;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QAEnB,QAAQ,GAAG,EAAE,CAAC;YACZ,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI;gBACP,OAAO,CAAC,IAAI,GAAG,IAAI,CAAA;gBACnB,MAAK;YAEP,KAAK,eAAe;gBAClB,0BAA0B;gBAC1B,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,UAAU,EAAE,CAAC;oBAC/B,OAAO,CAAC,MAAM,GAAG,oBAAoB,CAAA;oBACrC,CAAC,EAAE,CAAA,CAAC,gBAAgB;gBACtB,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,MAAM,GAAG,qBAAqB,CAAA;gBACxC,CAAC;gBACD,MAAK;YAEP,KAAK,SAAS;gBACZ,uCAAuC;gBACvC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,WAAW,EAAE,CAAC;oBAChC,OAAO,CAAC,MAAM,GAAG,eAAe,CAAA;oBAChC,CAAC,EAAE,CAAA,CAAC,gBAAgB;gBACtB,CAAC;qBAAM,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,UAAU,EAAE,CAAC;oBACtC,OAAO,CAAC,MAAM,GAAG,cAAc,CAAA;oBAC/B,CAAC,EAAE,CAAA,CAAC,gBAAgB;gBACtB,CAAC;qBAAM,CAAC;oBACN,gDAAgD;oBAChD,OAAO,CAAC,MAAM,GAAG,cAAc,CAAA;gBACjC,CAAC;gBACD,MAAK;YAEP,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAA;oBACvD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACjB,CAAC;gBACD,OAAO,CAAC,MAAM,GAAG,QAAQ,CAAA;gBACzB,OAAO,CAAC,UAAU,GAAG,OAAO,CAAA;gBAC5B,CAAC,EAAE,CAAA,CAAC,gBAAgB;gBACpB,MAAK;YACP,CAAC;YAED;gBACE,IAAI,GAAG,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzB,OAAO,CAAC,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAA;oBACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACjB,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,+CAA+C;AAC/C,eAAe;AACf,+CAA+C;AAE/C,SAAS,SAAS;IAChB,OAAO,CAAC,GAAG,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgCb,CAAC,CAAA;AACF,CAAC;AAED,+CAA+C;AAC/C,eAAe;AACf,+CAA+C;AAE/C,SAAS,aAAa,CAAC,OAAgB;IACrC,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;YACxB,OAAO,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAA;YACjD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;QACD,OAAO,IAAA,mBAAO,EAAC,OAAO,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;IACrD,CAAC;IAED,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;AAChC,CAAC;AAED,SAAS,OAAO,CAAC,UAAkB;IACjC,sBAAsB;IACtB,IAAI,CAAC,IAAA,oBAAU,EAAC,aAAa,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,KAAK,CAAC,qCAAqC,aAAa,EAAE,CAAC,CAAA;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,0BAA0B;IAC1B,MAAM,SAAS,GAAG,IAAA,mBAAO,EAAC,UAAU,CAAC,CAAA;IACrC,IAAI,CAAC,IAAA,oBAAU,EAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,IAAA,mBAAS,EAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACzC,OAAO,CAAC,GAAG,CAAC,sBAAsB,SAAS,EAAE,CAAC,CAAA;IAChD,CAAC;IAED,cAAc;IACd,IAAA,gBAAM,EAAC,aAAa,EAAE,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACtD,OAAO,CAAC,GAAG,CAAC,wBAAwB,UAAU,EAAE,CAAC,CAAA;AACnD,CAAC;AAED,+CAA+C;AAC/C,OAAO;AACP,+CAA+C;AAE/C,SAAS,IAAI;IACX,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAElC,6BAA6B;IAC7B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,SAAS,EAAE,CAAA;QACX,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;IAE/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,SAAS,EAAE,CAAA;QACX,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,CAAA;IAEzC,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAA;IACjD,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IACxC,OAAO,CAAC,GAAG,CAAC,SAAS,UAAU,EAAE,CAAC,CAAA;IAClC,OAAO,CAAC,GAAG,EAAE,CAAA;IAEb,OAAO,CAAC,UAAU,CAAC,CAAA;IAEnB,OAAO,CAAC,GAAG,EAAE,CAAA;IACb,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAA;IACrC,OAAO,CAAC,GAAG,EAAE,CAAA;IACb,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAA;IACtD,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAA;IAC3C,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAA;IAC/C,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAA;IACnD,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAA;AACpD,CAAC;AAED,IAAI,EAAE,CAAA"}

package/dist/embedder/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedder/index.ts"],"names":[],"mappings":"AAQA;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,6BAA6B;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAA;CACjB;AAMD;;GAEG;AACH,qBAAa,cAAe,SAAQ,KAAK;aAGZ,KAAK,CAAC,EAAE,KAAK;gBADtC,OAAO,EAAE,MAAM,EACU,KAAK,CAAC,EAAE,KAAK,YAAA;CAKzC;AAMD;;;;;;;GAOG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,KAAK,CAAoD;IACjE,OAAO,CAAC,WAAW,CAA6B;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,MAAM,EAAE,cAAc;IAIlC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAsBjC;;;OAGG;YACW,iBAAiB;IA+B/B;;;;;OAKG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAiC5C;;;;;OAKG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CA0BvD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedder/index.ts"],"names":[],"mappings":"AAQA;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,6BAA6B;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAA;CACjB;AAMD;;GAEG;AACH,qBAAa,cAAe,SAAQ,KAAK;aAGZ,KAAK,CAAC,EAAE,KAAK;gBADtC,OAAO,EAAE,MAAM,EACU,KAAK,CAAC,EAAE,KAAK,YAAA;CAKzC;AAMD;;;;;;;GAOG;AACH,qBAAa,QAAQ;IAEnB,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,WAAW,CAA6B;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,MAAM,EAAE,cAAc;IAIlC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAuBjC;;;OAGG;YACW,iBAAiB;IA+B/B;;;;;OAKG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAiC5C;;;;;OAKG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CA0BvD"}

package/dist/embedder/index.js

@@ -30,6 +30,7 @@ exports.EmbeddingError = EmbeddingError;
  */
 class Embedder {
     constructor(config) {
+        // Using unknown to avoid TS2590 (union type too complex with @types/jsdom)
         this.model = null;
         this.initPromise = null;
         this.config = config;
@@ -47,6 +48,7 @@ class Embedder {
             transformers_1.env.cacheDir = this.config.cacheDir;
             console.error(`Embedder: Setting cache directory to "${this.config.cacheDir}"`);
             console.error(`Embedder: Loading model "${this.config.modelPath}"...`);
+            // Use type assertion to avoid TS2590 (union type too complex with @types/jsdom)
             this.model = await (0, transformers_1.pipeline)('feature-extraction', this.config.modelPath);
             console.error('Embedder: Model loaded successfully');
         }

package/dist/embedder/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedder/index.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAE/C,4DAAyD;AAkBzD,+CAA+C;AAC/C,gBAAgB;AAChB,+CAA+C;AAE/C;;GAEG;AACH,MAAa,cAAe,SAAQ,KAAK;IACvC,YACE,OAAe,EACU,KAAa;QAEtC,KAAK,CAAC,OAAO,CAAC,CAAA;QAFW,UAAK,GAAL,KAAK,CAAQ;QAGtC,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAA;IAC9B,CAAC;CACF;AARD,wCAQC;AAED,+CAA+C;AAC/C,iBAAiB;AACjB,+CAA+C;AAE/C;;;;;;;GAOG;AACH,MAAa,QAAQ;IAKnB,YAAY,MAAsB;QAJ1B,UAAK,GAAgD,IAAI,CAAA;QACzD,gBAAW,GAAyB,IAAI,CAAA;QAI9C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,8BAA8B;QAC9B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAM;QACR,CAAC;QAED,IAAI,CAAC;YACH,+CAA+C;YAC/C,kBAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAA;YAEnC,OAAO,CAAC,KAAK,CAAC,yCAAyC,IAAI,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAA;YAC/E,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,CAAC,SAAS,MAAM,CAAC,CAAA;YACtE,IAAI,CAAC,KAAK,GAAG,MAAM,IAAA,uBAAQ,EAAC,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACxE,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACtD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,cAAc,CACtB,kCAAmC,KAAe,CAAC,OAAO,EAAE,EAC5D,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB;QAC7B,sBAAsB;QACtB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAM;QACR,CAAC;QAED,kDAAkD;QAClD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,WAAW,CAAA;YACtB,OAAM;QACR,CAAC;QAED,uBAAuB;QACvB,OAAO,CAAC,KAAK,CACX,+FAA+F,CAChG,CAAA;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACnD,8CAA8C;YAC9C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;YAEvB,+CAA+C;YAC/C,MAAM,IAAI,cAAc,CACtB,+CAAgD,KAAe,CAAC,OAAO,wTAAwT,IAAI,CAAC,MAAM,CAAC,QAAQ,gCAAgC,EACnb,KAAc,CACf,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,CAAC,WAAW,CAAA;IACxB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,0EAA0E;QAC1E,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,CAAC;YACH,mEAAmE;YACnE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,cAAc,CAAC,0CAA0C,CAAC,CAAA;YACtE,CAAC;YAED,uEAAuE;YACvE,8FAA8F;YAC9F,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;YACpD,MAAM,SAAS,GAAG,IAAI,CAAC,KAGa,CAAA;YACpC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YAE7C,qCAAqC;YACrC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACzC,OAAO,SAAS,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;gBACpC,MAAM,KAAK,CAAA;YACb,CAAC;YACD,MAAM,IAAI,cAAc,CACtB,iCAAkC,KAAe,CAAC,OAAO,EAAE,EAC3D,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,0EAA0E;QAC1E,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAA;QACX,CAAC;QAED,IAAI,CAAC;YACH,MAAM,UAAU,GAAe,EAAE,CAAA;YAEjC,6CAA6C;YAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBAC7D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;gBACvD,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBAChF,UAAU,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAA;YACrC,CAAC;YAED,OAAO,UAAU,CAAA;QACnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,cAAc,CACtB,wCAAyC,KAAe,CAAC,OAAO,EAAE,EAClE,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;CACF;AA5ID,4BA4IC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedder/index.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAE/C,4DAAyD;AAkBzD,+CAA+C;AAC/C,gBAAgB;AAChB,+CAA+C;AAE/C;;GAEG;AACH,MAAa,cAAe,SAAQ,KAAK;IACvC,YACE,OAAe,EACU,KAAa;QAEtC,KAAK,CAAC,OAAO,CAAC,CAAA;QAFW,UAAK,GAAL,KAAK,CAAQ;QAGtC,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAA;IAC9B,CAAC;CACF;AARD,wCAQC;AAED,+CAA+C;AAC/C,iBAAiB;AACjB,+CAA+C;AAE/C;;;;;;;GAOG;AACH,MAAa,QAAQ;IAMnB,YAAY,MAAsB;QALlC,2EAA2E;QACnE,UAAK,GAAY,IAAI,CAAA;QACrB,gBAAW,GAAyB,IAAI,CAAA;QAI9C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,8BAA8B;QAC9B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAM;QACR,CAAC;QAED,IAAI,CAAC;YACH,+CAA+C;YAC/C,kBAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAA;YAEnC,OAAO,CAAC,KAAK,CAAC,yCAAyC,IAAI,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAA;YAC/E,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,CAAC,SAAS,MAAM,CAAC,CAAA;YACtE,gFAAgF;YAChF,IAAI,CAAC,KAAK,GAAG,MAAM,IAAA,uBAAQ,EAAC,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACxE,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACtD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,cAAc,CACtB,kCAAmC,KAAe,CAAC,OAAO,EAAE,EAC5D,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB;QAC7B,sBAAsB;QACtB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAM;QACR,CAAC;QAED,kDAAkD;QAClD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,WAAW,CAAA;YACtB,OAAM;QACR,CAAC;QAED,uBAAuB;QACvB,OAAO,CAAC,KAAK,CACX,+FAA+F,CAChG,CAAA;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACnD,8CAA8C;YAC9C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;YAEvB,+CAA+C;YAC/C,MAAM,IAAI,cAAc,CACtB,+CAAgD,KAAe,CAAC,OAAO,wTAAwT,IAAI,CAAC,MAAM,CAAC,QAAQ,gCAAgC,EACnb,KAAc,CACf,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,CAAC,WAAW,CAAA;IACxB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,0EAA0E;QAC1E,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,CAAC;YACH,mEAAmE;YACnE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,cAAc,CAAC,0CAA0C,CAAC,CAAA;YACtE,CAAC;YAED,uEAAuE;YACvE,8FAA8F;YAC9F,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;YACpD,MAAM,SAAS,GAAG,IAAI,CAAC,KAGa,CAAA;YACpC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YAE7C,qCAAqC;YACrC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACzC,OAAO,SAAS,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;gBACpC,MAAM,KAAK,CAAA;YACb,CAAC;YACD,MAAM,IAAI,cAAc,CACtB,iCAAkC,KAAe,CAAC,OAAO,EAAE,EAC3D,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,0EAA0E;QAC1E,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAA;QACX,CAAC;QAED,IAAI,CAAC;YACH,MAAM,UAAU,GAAe,EAAE,CAAA;YAEjC,6CAA6C;YAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBAC7D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;gBACvD,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBAChF,UAAU,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAA;YACrC,CAAC;YAED,OAAO,UAAU,CAAA;QACnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,cAAc,CACtB,wCAAyC,KAAe,CAAC,OAAO,EAAE,EAClE,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;CACF;AA9ID,4BA8IC"}

package/dist/parser/html-parser.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Parse HTML content and extract main content as Markdown
+ *
+ * Flow:
+ * 1. HTML string → JSDOM (DOM creation)
+ * 2. JSDOM → Readability (main content extraction, noise removal)
+ * 3. Readability result → Turndown (Markdown conversion)
+ *
+ * @param html - Raw HTML string
+ * @param url - Source URL (used for resolving relative links)
+ * @returns Markdown string of extracted content
+ */
+export declare function parseHtml(html: string, url: string): Promise<string>;
+//# sourceMappingURL=html-parser.d.ts.map

package/dist/parser/html-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-parser.d.ts","sourceRoot":"","sources":["../../src/parser/html-parser.ts"],"names":[],"mappings":"AAsDA;;;;;;;;;;;GAWG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAoD1E"}

package/dist/parser/html-parser.js

@@ -0,0 +1,99 @@
+"use strict";
+// HTML Parser using Readability and Turndown
+// Extracts main content from HTML and converts to Markdown
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseHtml = parseHtml;
+const readability_1 = require("@mozilla/readability");
+const jsdom_1 = require("jsdom");
+const turndown_1 = __importDefault(require("turndown"));
+// ============================================
+// Turndown Service Configuration
+// ============================================
+/**
+ * Create and configure Turndown service for HTML to Markdown conversion
+ */
+function createTurndownService() {
+    const turndownService = new turndown_1.default({
+        headingStyle: 'atx', // Use # style headings
+        codeBlockStyle: 'fenced', // Use ``` for code blocks
+        bulletListMarker: '-', // Use - for bullet lists
+        emDelimiter: '_', // Use _ for emphasis
+        strongDelimiter: '**', // Use ** for bold
+    });
+    // Keep code blocks intact
+    turndownService.addRule('codeBlocks', {
+        filter: ['pre'],
+        replacement: (_content, node) => {
+            const element = node;
+            const codeElement = element.querySelector('code');
+            const code = codeElement ? codeElement.textContent : element.textContent;
+            const language = codeElement?.className?.replace('language-', '') || '';
+            return `\n\`\`\`${language}\n${code?.trim() || ''}\n\`\`\`\n`;
+        },
+    });
+    return turndownService;
+}
+// ============================================
+// HTML Parser
+// ============================================
+/**
+ * Parse HTML content and extract main content as Markdown
+ *
+ * Flow:
+ * 1. HTML string → JSDOM (DOM creation)
+ * 2. JSDOM → Readability (main content extraction, noise removal)
+ * 3. Readability result → Turndown (Markdown conversion)
+ *
+ * @param html - Raw HTML string
+ * @param url - Source URL (used for resolving relative links)
+ * @returns Markdown string of extracted content
+ */
+async function parseHtml(html, url) {
+    // Handle empty or whitespace-only HTML
+    if (!html || html.trim().length === 0) {
+        return '';
+    }
+    try {
+        // Create DOM from HTML string
+        const dom = new jsdom_1.JSDOM(html, {
+            url,
+            // Enable features needed for Readability
+            runScripts: 'outside-only',
+        });
+        const document = dom.window.document;
+        // Use Readability to extract main content
+        const reader = new readability_1.Readability(document, {
+            keepClasses: false,
+            debug: false,
+        });
+        const article = reader.parse();
+        // If Readability couldn't extract content, fall back to body text
+        if (!article || !article.content) {
+            // Try to get body content directly
+            const bodyContent = document.body?.innerHTML || '';
+            if (!bodyContent.trim()) {
+                return '';
+            }
+            // Convert raw body HTML to Markdown
+            const turndownService = createTurndownService();
+            return turndownService.turndown(bodyContent).trim();
+        }
+        // Convert extracted HTML content to Markdown
+        const turndownService = createTurndownService();
+        const markdown = turndownService.turndown(article.content);
+        // Add title if available
+        if (article.title) {
+            return `# ${article.title}\n\n${markdown}`.trim();
+        }
+        return markdown.trim();
+    }
+    catch (error) {
+        // Log error but don't throw - return empty string for graceful degradation
+        console.error('Failed to parse HTML:', error);
+        return '';
+    }
+}
+//# sourceMappingURL=html-parser.js.map

package/dist/parser/html-parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-parser.js","sourceRoot":"","sources":["../../src/parser/html-parser.ts"],"names":[],"mappings":";AAAA,6CAA6C;AAC7C,2DAA2D;;;;;AAiE3D,8BAoDC;AAnHD,sDAAkD;AAClD,iCAA6B;AAC7B,wDAAsC;AActC,+CAA+C;AAC/C,iCAAiC;AACjC,+CAA+C;AAE/C;;GAEG;AACH,SAAS,qBAAqB;IAC5B,MAAM,eAAe,GAAG,IAAI,kBAAe,CAAC;QAC1C,YAAY,EAAE,KAAK,EAAE,uBAAuB;QAC5C,cAAc,EAAE,QAAQ,EAAE,0BAA0B;QACpD,gBAAgB,EAAE,GAAG,EAAE,yBAAyB;QAChD,WAAW,EAAE,GAAG,EAAE,qBAAqB;QACvC,eAAe,EAAE,IAAI,EAAE,kBAAkB;KAC1C,CAAC,CAAA;IAEF,0BAA0B;IAC1B,eAAe,CAAC,OAAO,CAAC,YAAY,EAAE;QACpC,MAAM,EAAE,CAAC,KAAK,CAAC;QACf,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,OAAO,GAAG,IAAe,CAAA;YAC/B,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAA;YACjD,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAA;YACxE,MAAM,QAAQ,GAAG,WAAW,EAAE,SAAS,EAAE,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,IAAI,EAAE,CAAA;YACvE,OAAO,WAAW,QAAQ,KAAK,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,CAAA;QAC/D,CAAC;KACF,CAAC,CAAA;IAEF,OAAO,eAAe,CAAA;AACxB,CAAC;AAED,+CAA+C;AAC/C,cAAc;AACd,+CAA+C;AAE/C;;;;;;;;;;;GAWG;AACI,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,GAAW;IACvD,uCAAuC;IACvC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,CAAA;IACX,CAAC;IAED,IAAI,CAAC;QACH,8BAA8B;QAC9B,MAAM,GAAG,GAAG,IAAI,aAAK,CAAC,IAAI,EAAE;YAC1B,GAAG;YACH,yCAAyC;YACzC,UAAU,EAAE,cAAc;SAC3B,CAAC,CAAA;QAEF,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAA;QAEpC,0CAA0C;QAC1C,MAAM,MAAM,GAAG,IAAI,yBAAW,CAAC,QAAQ,EAAE;YACvC,WAAW,EAAE,KAAK;YAClB,KAAK,EAAE,KAAK;SACb,CAAC,CAAA;QAEF,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAA8B,CAAA;QAE1D,kEAAkE;QAClE,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACjC,mCAAmC;YACnC,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,IAAI,EAAE,CAAA;YAClD,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;gBACxB,OAAO,EAAE,CAAA;YACX,CAAC;YAED,oCAAoC;YACpC,MAAM,eAAe,GAAG,qBAAqB,EAAE,CAAA;YAC/C,OAAO,eAAe,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,CAAA;QACrD,CAAC;QAED,6CAA6C;QAC7C,MAAM,eAAe,GAAG,qBAAqB,EAAE,CAAA;QAC/C,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QAE1D,yBAAyB;QACzB,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,OAAO,KAAK,OAAO,CAAC,KAAK,OAAO,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAA;QACnD,CAAC;QAED,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAA;IACxB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,2EAA2E;QAC3E,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,KAAK,CAAC,CAAA;QAC7C,OAAO,EAAE,CAAA;IACX,CAAC;AACH,CAAC"}

package/dist/server/index.d.ts

@@ -1,4 +1,5 @@
 import { type GroupingMode } from '../vectordb/index.js';
+import { type ContentFormat } from './raw-data-utils.js';
 /**
  * RAGServer configuration
  */
@@ -36,12 +37,33 @@ export interface IngestFileInput {
     /** File path */
     filePath: string;
 }
+/**
+ * ingest_data tool input metadata
+ */
+export interface IngestDataMetadata {
+    /** Source identifier: URL ("https://...") or custom ID ("clipboard://2024-12-30") */
+    source: string;
+    /** Content format */
+    format: ContentFormat;
+}
+/**
+ * ingest_data tool input
+ */
+export interface IngestDataInput {
+    /** Content to ingest (text, HTML, or Markdown) */
+    content: string;
+    /** Content metadata */
+    metadata: IngestDataMetadata;
+}
 /**
  * delete_file tool input
+ * Either filePath or source must be provided
  */
 export interface DeleteFileInput {
-    /** File path */
-    filePath: string;
+    /** File path (for files ingested via ingest_file) */
+    filePath?: string;
+    /** Source identifier (for data ingested via ingest_data) */
+    source?: string;
 }
 /**
  * ingest_file tool output
@@ -66,6 +88,8 @@ export interface QueryResult {
     text: string;
     /** Similarity score */
     score: number;
+    /** Original source (only for raw-data files, e.g., URLs ingested via ingest_data) */
+    source?: string;
 }
 /**
  * RAG server compliant with MCP Protocol
@@ -82,6 +106,7 @@ export declare class RAGServer {
     private readonly embedder;
     private readonly chunker;
     private readonly parser;
+    private readonly dbPath;
     constructor(config: RAGServerConfig);
     /**
      * Set up MCP handlers
@@ -110,7 +135,23 @@ export declare class RAGServer {
         }];
     }>;
     /**
-     * list_files tool handler (Phase 1: basic implementation)
+     * ingest_data tool handler
+     * Saves raw content to raw-data directory and calls handleIngestFile internally
+     *
+     * For HTML content:
+     * - Parses HTML and extracts main content using Readability
+     * - Converts to Markdown for better chunking
+     * - Saves as .md file
+     */
+    handleIngestData(args: IngestDataInput): Promise<{
+        content: [{
+            type: 'text';
+            text: string;
+        }];
+    }>;
+    /**
+     * list_files tool handler
+     * Enriches raw-data files with original source information
      */
     handleListFiles(): Promise<{
         content: [{
@@ -129,6 +170,8 @@ export declare class RAGServer {
     }>;
     /**
      * delete_file tool handler
+     * Deletes chunks from VectorDB and physical raw-data files
+     * Supports both filePath (for ingest_file) and source (for ingest_data)
      */
     handleDeleteFile(args: DeleteFileInput): Promise<{
         content: [{

package/dist/server/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,KAAK,YAAY,EAAiC,MAAM,sBAAsB,CAAA;AAMvF;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAA;IACd,iCAAiC;IACjC,SAAS,EAAE,MAAM,CAAA;IACjB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,8BAA8B;IAC9B,OAAO,EAAE,MAAM,CAAA;IACf,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAA;IACnB,kEAAkE;IAClE,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,qDAAqD;IACrD,QAAQ,CAAC,EAAE,YAAY,CAAA;IACvB,sFAAsF;IACtF,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,iDAAiD;IACjD,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,gBAAgB;IAChB,SAAS,EAAE,MAAM,CAAA;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,uBAAuB;IACvB,KAAK,EAAE,MAAM,CAAA;CACd;AAMD;;;;;;;;GAQG;AACH,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAa;IACzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAU;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiB;IACzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,MAAM,EAAE,eAAe;IAoCnC;;OAEG;IACH,OAAO,CAAC,aAAa;IAmGrB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAKjC;;OAEG;IACG,oBAAoB,CACxB,IAAI,EAAE,mBAAmB,GACxB,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IA8BzD;;OAEG;IACG,gBAAgB,CACpB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IA0HzD;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAiB/E;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAiB5E;;OAEG;IACG,gBAAgB,CACpB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAoCzD;;OAEG;IACG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAK3B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,KAAK,YAAY,EAAiC,MAAM,sBAAsB,CAAA;AACvF,OAAO,EACL,KAAK,aAAa,EAKnB,MAAM,qBAAqB,CAAA;AAM5B;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAA;IACd,iCAAiC;IACjC,SAAS,EAAE,MAAM,CAAA;IACjB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,8BAA8B;IAC9B,OAAO,EAAE,MAAM,CAAA;IACf,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAA;IACnB,kEAAkE;IAClE,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,qDAAqD;IACrD,QAAQ,CAAC,EAAE,YAAY,CAAA;IACvB,sFAAsF;IACtF,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,iDAAiD;IACjD,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,qFAAqF;IACrF,MAAM,EAAE,MAAM,CAAA;IACd,qBAAqB;IACrB,MAAM,EAAE,aAAa,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,kDAAkD;IAClD,OAAO,EAAE,MAAM,CAAA;IACf,uBAAuB;IACvB,QAAQ,EAAE,kBAAkB,CAAA;CAC7B;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,qDAAqD;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,4DAA4D;IAC5D,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,gBAAgB;IAChB,SAAS,EAAE,MAAM,CAAA;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,uBAAuB;IACvB,KAAK,EAAE,MAAM,CAAA;IACb,qFAAqF;IACrF,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAMD;;;;;;;;GAQG;AACH,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAa;IACzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAU;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiB;IACzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;gBAEnB,MAAM,EAAE,eAAe;IAqCnC;;OAEG;IACH,OAAO,CAAC,aAAa;IA0IrB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAKjC;;OAEG;IACG,oBAAoB,CACxB,IAAI,EAAE,mBAAmB,GACxB,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IA0CzD;;OAEG;IACG,gBAAgB,CACpB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAmIzD;;;;;;;;OAQG;IACG,gBAAgB,CACpB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAyDzD;;;OAGG;IACG,eAAe,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IA6B/E;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAiB5E;;;;OAIG;IACG,gBAAgB,CACpB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IA+DzD;;OAEG;IACG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAK3B"}