@lotargo/memory_plugin 1.4.621 → 1.5.0

package/package.json

@@ -1,11 +1,13 @@
 {
   "name": "@lotargo/memory_plugin",
-  "version": "1.4.621",
+  "version": "1.5.0",
   "description": "100% local hybrid RAG memory for AI coding agents (OpenCode, Claude Code, Codex, Antigravity). MCP server + plugin: persistent user facts, document ingestion, vector + SQLite FTS5 retrieval across sessions.",
   "type": "module",
   "main": "opencode-plugin/index.js",
   "scripts": {
-    "preinstall": "node mcp-server/preinstall.js || true"
+    "preinstall": "node mcp-server/preinstall.js || true",
+    "test": "node mcp-server/test.js",
+    "benchmark": "node mcp-server/benchmarks/run_benchmarks.js"
   },
   "bin": {
     "memory_plugin": "mcp-server/index.js",
@@ -15,7 +17,7 @@
   "files": [
     "opencode-plugin",
     "mcp-server/admin",
-    "mcp-server/benchmarks",
+    "mcp-server/cli",
     "mcp-server/config",
     "mcp-server/db",
     "mcp-server/graph",
@@ -23,9 +25,11 @@
     "mcp-server/ml",
     "mcp-server/retrieval",
     "mcp-server/storage",
+    "mcp-server/tools",
     "mcp-server/cli.js",
     "mcp-server/index.js",
     "mcp-server/fact_format.js",
+    "mcp-server/identity.js",
     "mcp-server/memory.js",
     "mcp-server/setup.js",
     "mcp-server/preinstall.js",

package/skills/using-memory/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: using-memory
-description: Comprehensive guide for using the Memory, Hybrid RAG Knowledge Engine & MCP Helper tools (remember, recall, forget, update_fact, memory_info, link_knowledge, ingest_document, query_knowledge_base, manage_knowledge_base, list-mcp-tools, mcp-reminder). Trigger proactively whenever user preferences, project conventions, technology stack choices, or architecture decisions are introduced, or when querying ingested documentation, indexing files/repos, managing persistent knowledge, or looking up available MCP tool integrations.
+description: Comprehensive guide for using the Memory, Hybrid RAG Knowledge Engine & MCP Helper tools (remember, recall, get_fact, forget, update_fact, memory_info, link_knowledge, link_project_memory, unlink_project_memory, relink_project_memory, ingest_document, query_knowledge_base, manage_knowledge_base, list-mcp-tools, mcp-reminder). Trigger proactively whenever user preferences, project conventions, technology stack choices, or architecture decisions are introduced, or when querying ingested documentation, indexing files/repos, managing persistent knowledge, or looking up available MCP tool integrations.
 ---
 
 # Using Memory, Hybrid RAG Knowledge Engine & MCP Helper Tools
@@ -17,19 +17,23 @@ You have access to a persistent dual-layer memory engine supercharged with an **
 
 | Scenario / Intent | Target Tool | Key Parameters |
 |-------------------|-------------|----------------|
-| User shares identity, tech stack preference, or workflow rule | `remember` | `fact` (English), `scope`, optional `docId`, `startLine`, `endLine` |
-| User asks what you remember about them, the project, or linked docs | `recall` | `scope` ("all", "global", or "project"), optional `query`, `tags`, `since`, `until`, `project` |
+| User shares identity, tech stack preference, or workflow rule | `remember` | `fact` (English), `title` (concise 2-5 word headline), `scope`, optional `docId`, `startLine`, `endLine` |
+| User asks what you remember about them, the project, or linked docs | `recall` | `scope` ("all", "global", "project", "list_projects"), `mode` ("full", "headers"), `offset`, `limit`, optional `query`, `tags`, `since`, `until`, `project` (at session start, MUST fetch all memories with `scope: "all"` without restrictive query filters) |
+| Get a single fact's text and metadata by ID | `get_fact` | `id` (metadata id e.g. "8f3a2c"), `scope` |
 | User corrects/updates an old saved fact | `update_fact` | `id` (number/id/text), `newText`, `scope` |
 | Replace a fact but keep a version trail | `remember` | `fact`, `supersedes` (number/id/text) |
 | Protect a fact from accidental `forget` | `remember` | `keep: true` |
 | Set a time-to-live on a fact | `remember` | `ttl` ("90d", "2w", "24h", "12m") |
 | Filter facts by keyword / tags / date | `recall` | `query`, `tags`, `since`, `until` |
-| Show storage paths, versions, fact & RAG stats | `memory_info` | — |
-| Connect a Notebook fact to a document, section, or line range | `link_knowledge` | `factText`, `docId`, `startLine`, `endLine`, `relationType` |
-| User asks to index a documentation URL, file, or repository | `ingest_document` | `content` or `source_path`, `title`, `metadata` |
-| User asks a complex question about indexed docs or code | `query_knowledge_base` | `query`, `limit`, `generateEmbeddings` |
+| Show storage paths, versions, fact & RAG stats, git identity | `memory_info` | — |
+| Connect a Notebook fact to a document, section, or line range | `link_knowledge` | `action` ("link", "list_links", "get_doc_links"), `factText`, `docId`, `startLine`, `endLine`, `relationType` |
+| Link directory to Git project identity / migrate legacy stores | `link_project_memory` | `directory`, optional `remote` |
+| Remove path alias or purge project identity | `unlink_project_memory` | `directory`, `purge` (boolean) |
+| Move or merge project memories to new target identity | `relink_project_memory` | `directory`, `remote` (target remote URL) |
+| User asks to index a documentation URL, file, or repository | `ingest_document` | `content` (text/file path/URL), `type` ("text", "file", "url"), `title`, `path` |
+| User asks a complex question about indexed docs or code | `query_knowledge_base` | `query`, `limit`, `instruction`, `generateEmbeddings` |
 | Read full raw content of an ambiguous/abstract document | `manage_knowledge_base` | `action: "read_document"`, `docId` |
-| User asks to view database stats, list indexed docs, or export snapshots | `manage_knowledge_base` | `action` ("stats", "list", "read_document", "delete", "export_snapshot", "import_snapshot") |
+| View DB stats, list indexed docs, read/delete docs, export/import snapshots | `manage_knowledge_base` | `action` ("stats", "list", "read_document", "delete", "export_snapshot", "import_snapshot"), `docId`, `snapshotPath` |
 | Discover available MCP servers and their specific purposes | `list-mcp-tools` | — |
 | Ask which MCP tool / server is suitable for a specific task | `mcp-reminder` | `task` (string, e.g., "db migration") |
 
@@ -44,10 +48,13 @@ Whenever you ingest project documentation, web pages, or local files, you should
 
 ### What to Save and Link (`remember` & `link_knowledge`)
 - **High-Signal Facts**: User name, role, language preferences, architectural constraints, framework choices, coding standards, test rules.
-- **Formating**: Always translate the fact into clear, concise English before calling `remember`.
+- **Formatting & Fact Titles**:
+  - Always translate the fact into clear, concise English before calling `remember`.
+  - **Always specify a descriptive `title` parameter** (a 2-5 word headline, e.g., `title: "Backend Framework Preference"`).
+  - Facts are stored in `**Title** — body` format. In `mode: "full"` (default in `recall`), both title and body are displayed. In `mode: "headers"` and in auto-injected `<MEMORY>` system prompt blocks, only `**Title**` is displayed to keep the system prompt lean.
 - **Linking to Knowledge Base Documents**:
   - Pass `docId` (or document title/path) and optional `startLine` / `endLine` when calling `remember` or `link_knowledge`.
-  - Example: `remember(fact: "Use Fastify instead of Express for backend services", scope: "project", docId: "arch_specs.md", startLine: 5, endLine: 7)`
+  - Example: `remember(title: "Backend Framework Preference", fact: "Use Fastify instead of Express for backend services", scope: "project", docId: "arch_specs.md", startLine: 5, endLine: 7)`
   - Example: `link_knowledge(factText: "Use PostgreSQL 16 for primary persistence", docId: "database_guide.md", startLine: 20, endLine: 35, relationType: "IMPLEMENTS")`
 
 ### How Linked Memory Appears (`recall`)
@@ -76,12 +83,16 @@ Supported metadata keys (set via `remember`, rendered as badges by `recall`):
 - `tags`: comma-separated tags for later filtering, e.g. `"pref,arch"`.
 - `supersedes`: number (as listed by `recall`), metadata `id`, or text of the fact this one replaces.
 
-### Filtering Facts (`recall`)
+### Filtering & Viewing Facts (`recall` & `get_fact`)
+- `scope`: `"all"` (default), `"global"`, `"project"`, or `"list_projects"` (lists all project stores, total facts, file paths, and git identity bindings).
 - `query`: all space-separated terms must match (case-insensitive); searches text, id, tags, and date.
 - `tags`: comma-separated; returns facts with ANY matching tag.
 - `since` / `until`: "YYYY-MM-DD" (inclusive) to filter by fact date.
 - `project`: read a specific project's store from any working directory.
-- Output shows `[EXPIRED]`, `[KEEP]`, `[SUPERSEDED]` badges and the `Store file:` path.
+- `mode`: `"full"` (default) or `"headers"` (returns title and badges only, omitting full text body).
+- `offset` / `limit`: optional numeric pagination parameters.
+- `get_fact`: fetch exact text and full metadata of a single fact by its metadata id (e.g. `get_fact(id: "8f3a2c")`).
+- Output shows `[EXPIRED]`, `[KEEP]`, `[SUPERSEDED]`, `[INJECT]` badges and the `Store file:` path.
 
 ### Updating Facts (`update_fact`)
 When the user corrects an old fact, prefer `update_fact` over `forget`+`remember` — it rewrites the text while preserving the original date and all metadata (`ttl`, `keep`, `tags`, `supersedes`), and re-points any linked Knowledge Base documents.
@@ -92,8 +103,14 @@ When the user corrects an old fact, prefer `update_fact` over `forget`+`remember
 ### Protecting Facts (`forget` with `keep`)
 `forget` refuses to delete facts saved with `keep: true`; pass `force: true` to override. It still supports deleting by index number, range ("3-30"), or text.
 
+### Project Memory Identity Management (`link_project_memory`, `unlink_project_memory`, `relink_project_memory`)
+Project stores are bound to Git-based project identities (`git:remote` or `git:local:<repo basename>`). Use these tools to manage bindings:
+- `link_project_memory(directory, remote)`: Links a working directory to a Git identity, registers path/remote aliases, and automatically merges any legacy path-based stores.
+- `unlink_project_memory(directory, purge)`: Removes the path alias link for a directory; set `purge: true` to purge the identity from SQLite.
+- `relink_project_memory(directory, remote)`: Moves and merges memories from the current project identity to a new target remote URL identity.
+
 ### Storage Diagnostics (`memory_info`)
-`memory_info` returns the package version, `MEMORY_DIR`, SQLite DB path, store-file locations, fact counts per store, and RAG stats (documents, sections, chunks, graph edges, links).
+`memory_info` returns the package version, `MEMORY_DIR`, SQLite DB path, store-file locations, fact counts per store, Git identity info, and RAG stats (documents, sections, chunks, graph edges, links).
 
 ---
 
@@ -176,7 +193,7 @@ When working in multi-server environments (e.g., OpenCode, Claude Code), you mig
 
 ## 5. Core Directives for AI Agents
 
-1. **Read Memories First (MANDATORY)**: At the very start of any session or conversation, your VERY FIRST STEP MUST BE to execute `recall` to load all saved facts, user context, and project guidelines BEFORE performing any other task or code analysis.
+1. **Read Memories First (MANDATORY)**: At the very start of any session or conversation, your VERY FIRST STEP MUST BE to execute `recall` with `scope: "all"` (and without restrictive query filters) to load ALL saved global facts, user context, and project guidelines BEFORE performing any other task or code analysis. Do NOT filter or restrict the initial `recall` call with specific query keywords.
 2. **Be Proactive**: When the user mentions a durable preference, personal fact, or constraint, save it immediately using `remember`. Do not wait for explicit user commands.
 3. **Check Knowledge Base First**: If a query is related to specialized documentation, APIs, or project architectures, call `query_knowledge_base` using concept-dense search phrases.
 4. **Inspect Ambiguous Docs Directly**: If querying produces low relevance scores on abstractly-named documents, call `manage_knowledge_base(action: "read_document")` to inspect the full text directly.

package/mcp-server/benchmarks/fetch_real_corpus.js

@@ -1,351 +0,0 @@
-import { readFile, readdir, stat, rm, mkdir as mkdirAsync, writeFile as writeFileAsync } from "node:fs/promises";
-import { existsSync } from "node:fs";
-import { join } from "node:path";
-import { MEMORY_DIR } from "../memory.js";
-
-export const CORPUS_DIR = join(MEMORY_DIR, "cache", "benchmark_corpus");
-const PANEL_WIDTH = 58;
-
-function printRichPanel(title, subtitle = "") {
-  const line = "─".repeat(PANEL_WIDTH - 2);
-  console.log(`\x1b[36m╭${line}╮\x1b[0m`);
-  console.log(`\x1b[36m│\x1b[0m  \x1b[1m\x1b[37m${title.padEnd(PANEL_WIDTH - 6)}\x1b[0m  \x1b[36m│\x1b[0m`);
-  if (subtitle) {
-    console.log(`\x1b[36m│\x1b[0m  \x1b[90m${subtitle.padEnd(PANEL_WIDTH - 6)}\x1b[0m  \x1b[36m│\x1b[0m`);
-  }
-  console.log(`\x1b[36m╰${line}╯\x1b[0m`);
-}
-
-const RAW_DOC_SOURCES = [
-  // Frontend frameworks & libraries
-  { id: "react_readme", title: "React README", url: "https://raw.githubusercontent.com/facebook/react/main/README.md" },
-  { id: "react_license", title: "React License", url: "https://raw.githubusercontent.com/facebook/react/main/LICENSE" },
-  { id: "vue_readme", title: "Vue.js README", url: "https://raw.githubusercontent.com/vuejs/core/main/README.md" },
-  { id: "svelte_readme", title: "Svelte README", url: "https://raw.githubusercontent.com/sveltejs/svelte/main/README.md" },
-  { id: "nextjs_readme", title: "Next.js README", url: "https://raw.githubusercontent.com/vercel/next.js/canary/packages/next/README.md" },
-  // Build tools & bundlers
-  { id: "vite_readme", title: "Vite README", url: "https://raw.githubusercontent.com/vitejs/vite/main/README.md" },
-  { id: "esbuild_readme", title: "esbuild README", url: "https://raw.githubusercontent.com/evanw/esbuild/main/README.md" },
-  { id: "webpack_readme", title: "Webpack README", url: "https://raw.githubusercontent.com/webpack/webpack/main/README.md" },
-  // Backend & runtime
-  { id: "express_readme", title: "Express.js README", url: "https://raw.githubusercontent.com/expressjs/express/master/Readme.md" },
-  { id: "fastapi_readme", title: "FastAPI README", url: "https://raw.githubusercontent.com/fastapi/fastapi/master/README.md" },
-  { id: "flask_readme", title: "Flask Python README", url: "https://raw.githubusercontent.com/pallets/flask/main/README.md" },
-  { id: "deno_readme", title: "Deno Engine README", url: "https://raw.githubusercontent.com/denoland/deno/main/README.md" },
-  { id: "bun_readme", title: "Bun Runtime README", url: "https://raw.githubusercontent.com/oven-sh/bun/main/README.md" },
-  // Languages
-  { id: "golang_readme", title: "Go Language README", url: "https://raw.githubusercontent.com/golang/go/master/README.md" },
-  { id: "rust_readme", title: "Rust Language README", url: "https://raw.githubusercontent.com/rust-lang/rust/master/README.md" },
-  { id: "typescript_readme", title: "TypeScript README", url: "https://raw.githubusercontent.com/microsoft/TypeScript/main/README.md" },
-  // State management & data
-  { id: "redux_readme", title: "Redux README", url: "https://raw.githubusercontent.com/reduxjs/redux/master/README.md" },
-  { id: "prisma_readme", title: "Prisma ORM README", url: "https://raw.githubusercontent.com/prisma/prisma/main/README.md" },
-  { id: "graphql_js_readme", title: "GraphQL.js README", url: "https://raw.githubusercontent.com/graphql/graphql-js/v16.8.1/README.md" },
-  // Databases
-  { id: "sqlite_readme", title: "SQLite README", url: "https://raw.githubusercontent.com/sqlite/sqlite/master/README.md" },
-  // CSS & styling
-  { id: "tailwindcss_readme", title: "Tailwind CSS README", url: "https://raw.githubusercontent.com/tailwindlabs/tailwindcss/master/README.md" },
-  // Testing
-  { id: "playwright_readme", title: "Playwright README", url: "https://raw.githubusercontent.com/microsoft/playwright/main/README.md" },
-  // Networking & HTTP
-  { id: "axios_readme", title: "Axios HTTP README", url: "https://raw.githubusercontent.com/axios/axios/v1.x/README.md" },
-  // Utilities
-  { id: "lodash_readme", title: "Lodash Utility README", url: "https://raw.githubusercontent.com/lodash/lodash/master/README.md" },
-  { id: "zod_readme", title: "Zod Validation README", url: "https://raw.githubusercontent.com/colinhacks/zod/master/packages/zod/README.md" },
-  // Visualization
-  { id: "chartjs_readme", title: "Chart.js Library README", url: "https://raw.githubusercontent.com/chartjs/Chart.js/master/README.md" },
-  { id: "threejs_readme", title: "Three.js README", url: "https://raw.githubusercontent.com/mrdoob/three.js/dev/README.md" },
-  // ML / AI
-  { id: "transformers_js_readme", title: "Transformers.js README", url: "https://raw.githubusercontent.com/xenova/transformers.js/main/README.md" },
-  // Infrastructure
-  { id: "docker_cli_readme", title: "Docker CLI README", url: "https://raw.githubusercontent.com/docker/cli/master/README.md" },
-  // Editor
-  { id: "vscode_readme", title: "VS Code README", url: "https://raw.githubusercontent.com/microsoft/vscode/main/README.md" },
-];
-
-export const LOCAL_FALLBACK_DOCS = [
-  {
-    id: "nextjs_readme",
-    title: "Next.js React Framework Architecture & Specifications",
-    content: `# Next.js: The React Framework for Full-Stack Web Applications
-
-Next.js is a progressive React framework for building full-stack web applications. It provides server-side rendering (SSR), static site generation (SSG), and client-side rendering with file-based routing.
-
-Key Features:
-- File-based routing system with the App Router (page, layout, loading, error handlers).
-- Server-side rendering (SSR) and React Server Components for optimal SEO and fast initial page loads.
-- Integrated CLI commands for application lifecycle:
-  - \`next dev\`: Starts the development server with Hot Module Replacement (HMR).
-  - \`next build\`: Compiles and optimizes the application for production deployment.
-  - \`next start\`: Runs the production server.
-  - \`npx create-next-app\`: Scaffolds a new Next.js application with TypeScript, Tailwind, and React Toolkit templates.
-- Built-in image, font, and script optimization.
-`,
-  },
-  {
-    id: "vue_readme",
-    title: "Vue.js 3 Core Architecture & Reactivity Specification",
-    content: `# Vue.js: Progressive JavaScript Framework
-
-Vue.js is a progressive JavaScript framework for building modern user interfaces based on a reactive data model.
-
-Core Principles:
-- Declarative rendering and reactive state management via Composition API (\`ref\`, \`reactive\`, \`computed\`).
-- Virtual DOM rendering pipeline with optimized diffing and fine-grained reactivity tracking.
-- Single File Components (SFC) combining template, script, and style in a single \`.vue\` file.
-- Built-in transitions, component lifecycle hooks, and lightweight core footprint.
-`,
-  },
-  {
-    id: "deno_readme",
-    title: "Deno Engine Runtime Architecture & Security Model",
-    content: `# Deno: Secure Runtime for JavaScript and TypeScript
-
-Deno is a modern, secure runtime for JavaScript and TypeScript built on V8, Rust, and Tokio.
-
-Security Sandbox & Network Permissions:
-- By default, execution is completely sandboxed: direct network access, file system access, and environment variable access are strictly banned (disallowed/restricted) unless explicitly granted by CLI permission flags.
-- Banning network permissions by default ensures untrusted TypeScript code cannot leak sensitive data.
-- Network and file system permission flags:
-  - \`--allow-net\`: Grants network access permissions to specific domains or all network interfaces.
-  - \`--allow-read\` & \`--allow-write\`: Grants restricted file system permissions.
-  - \`--allow-env\`: Grants environment variable access.
-- Native TypeScript and JSX support out of the box without external compilers or build configuration.
-- Single binary distribution with built-in test runner, formatter, and linter.
-`,
-  },
-  {
-    id: "rust_readme",
-    title: "Rust Systems Programming Language Specification",
-    content: `# Rust Programming Language
-
-Rust is a systems programming language focused on memory safety, concurrency, and high performance without relying on a Garbage Collector (GC).
-
-Core Mechanics:
-- Memory safety guarantees enforced at compile-time via Ownership, Borrowing, and Lifetimes.
-- Zero-cost abstractions providing C/C++ speed with memory-safe guarantees.
-- Elimination of data races in multi-threaded concurrent programming.
-- Package manager and build tool integration via Cargo.
-`,
-  },
-  {
-    id: "docker_cli_readme",
-    title: "Docker CLI & Container Management Specification",
-    content: `# Docker Command Line Interface (CLI)
-
-The Docker CLI provides command-line tools for creating, managing, and orchestrating isolated application containers.
-
-Key Concepts & Operations:
-- Isolated container execution environments separating applications from underlying host operating systems.
-- Container lifecycle commands: \`docker run\`, \`docker exec\`, \`docker stop\`, \`docker rm\`.
-- Image management, Dockerfile builds, and multi-container orchestration via Docker Compose.
-`,
-  },
-  {
-    id: "bun_readme",
-    title: "Bun Runtime Engine & Package Manager Specification",
-    content: `# Bun JavaScript Runtime
-
-Bun is a fast, all-in-one JavaScript runtime, bundler, test runner, and package manager designed as a drop-in replacement for Node.js.
-
-Key Features:
-- Native TypeScript and JSX support out of the box without transpilation steps.
-- High-performance Zig-based engine powered by WebKit JavaScriptCore.
-- Built-in package manager with ultra-fast dependency installation.
-`,
-  },
-  {
-    id: "redux_readme",
-    title: "Redux State Management Store Architecture",
-    content: `# Redux: Centralized Application State Management Store
-
-Redux is a predictable state container for JavaScript applications, providing centralized application state management within a single immutable store (централизованное управление состоянием приложения в одном сторе).
-
-Core Features:
-- Centralized application state management in a single store tree.
-- Predictable state mutations using pure reducer functions and dispatched action objects.
-- Redux Toolkit (RTK) with \`configureStore\`, \`createSlice\`, and RTK Query for data fetching.
-- DevTools integration for time-travel debugging and action inspection.
-`,
-  },
-  {
-    id: "zod_readme",
-    title: "Zod Schema Validation & TypeScript Type Inference Specification",
-    content: `# Zod: Declarative Schema Validation Library
-
-Zod is a TypeScript-first declarative schema validation library with automatic static type inference (декларативная валидация схем с автоматическим выводом типов TypeScript).
-
-Key Features:
-- Declarative schema validation library for objects, strings, numbers, arrays, and enums.
-- Automatic TypeScript type inference (\`z.infer<typeof schema>\`) deriving static TypeScript types directly from validation schemas.
-- Safe parsing via \`schema.safeParse()\` returning structured error results.
-`,
-  },
-  {
-    id: "sqlite_readme",
-    title: "SQLite Embedded Database Architecture",
-    content: `# SQLite Database Engine
-
-SQLite is a compact, self-contained, serverless, zero-configuration SQL database engine running directly inside the application process.
-
-Features:
-- In-process embedded execution without requiring a standalone database server daemon.
-- Write-Ahead Logging (WAL) mode for concurrent high-speed read/write performance.
-- Full-text search extension via FTS5 virtual table module.
-`,
-  },
-  {
-    id: "sqlite_fts5_spec",
-    title: "SQLite FTS5 & BM25 Full-Text Search Specification",
-    content: `# SQLite FTS5 Search Extension
-FTS5 is an SQLite virtual table module that provides full-text search capability.
-Features:
-- BM25 ranking algorithm calculation for keyword relevance.
-- Tokenization using standard unicode61 tokenizer.
-- Prefix searching, phrase queries, and boolean AND/OR operator support.
-- Ultra fast search performance under WAL (Write-Ahead Logging) mode.
-`,
-  },
-  {
-    id: "onnx_runtime_spec",
-    title: "ONNX Runtime Inference Engine Specification",
-    content: `# ONNX Runtime: Cross-Platform Machine Learning Model Accelerator
-ONNX Runtime is a cross-platform inference engine for machine learning models in the Open Neural Network Exchange (ONNX) format.
-Key components:
-- Quantized ONNX models (q8, q4) minimize RAM consumption.
-- Hardware-accelerated inference via CPU (MLAS), GPU (CUDA, DirectML, ROCm).
-- Node.js bindings via onnxruntime-node and onnxruntime-web packages.
-`,
-  },
-];
-
-const MIN_REAL_DOCS_BEFORE_FALLBACK = 15;
-const FETCH_CONCURRENCY = 6;
-
-async function fetchSingleDoc(doc, targetDir) {
-  const filePath = join(targetDir, `${doc.id}.md`);
-
-  // Use cached file if it already exists
-  if (existsSync(filePath)) {
-    try {
-      const content = await readFile(filePath, "utf-8");
-      if (content.length > 0) {
-        return { id: doc.id, title: doc.title, path: filePath, bytes: content.length, source: "cached" };
-      }
-    } catch {
-      // Cache read failed, re-fetch below
-    }
-  }
-
-  // Fetch from network
-  const response = await fetch(doc.url, { signal: AbortSignal.timeout(10000) });
-  if (!response.ok) {
-    throw new Error(`HTTP ${response.status}`);
-  }
-  const text = await response.text();
-  await writeFileAsync(filePath, text, "utf-8");
-  return { id: doc.id, title: doc.title, path: filePath, bytes: text.length, source: "network" };
-}
-
-export async function fetchRealCorpus({ silent = false, onProgress = null, subsetDocIds = null } = {}) {
-  if (!existsSync(CORPUS_DIR)) {
-    await mkdirAsync(CORPUS_DIR, { recursive: true });
-  }
-
-  if (!silent) {
-    printRichPanel(
-      "FETCHING TECHNICAL CORPUS",
-      subsetDocIds ? `Cache: ${CORPUS_DIR} (subset: ${subsetDocIds.length} docs)` : `Cache: ${CORPUS_DIR}`,
-    );
-  }
-
-  // When subsetDocIds is provided, filter RAW_DOC_SOURCES (and LOCAL_FALLBACK_DOCS
-  // below) to only the requested ids. Used by smoke mode to ingest a small subset
-  // for fast iteration instead of the full 27-doc corpus.
-  const docPool = subsetDocIds
-    ? RAW_DOC_SOURCES.filter((d) => subsetDocIds.includes(d.id))
-    : RAW_DOC_SOURCES;
-
-  const results = [];
-  let networkCount = 0;
-  let cachedCount = 0;
-  const total = docPool.length;
-
-  // Fetch in parallel batches for performance
-  for (let i = 0; i < docPool.length; i += FETCH_CONCURRENCY) {
-    const batch = docPool.slice(i, i + FETCH_CONCURRENCY);
-    const settled = await Promise.allSettled(
-      batch.map((doc) => fetchSingleDoc(doc, CORPUS_DIR))
-    );
-
-    for (const result of settled) {
-      if (result.status === "fulfilled") {
-        results.push(result.value);
-        if (result.value.source === "network") {
-          networkCount++;
-          if (!silent) console.log(`  [OK] Fetched ${result.value.id} (${result.value.bytes} bytes)`);
-        } else if (result.value.source === "cached") {
-          cachedCount++;
-        }
-      }
-    }
-
-    if (onProgress) onProgress({ phase: "fetch", current: Math.min(i + FETCH_CONCURRENCY, total), total });
-  }
-
-  // Ensure verified technical documentation specs overwrite stub files in corpus
-  for (const fallbackDoc of LOCAL_FALLBACK_DOCS) {
-    const filePath = join(CORPUS_DIR, `${fallbackDoc.id}.md`);
-    await writeFileAsync(filePath, fallbackDoc.content, "utf-8");
-    const idx = results.findIndex((r) => r.id === fallbackDoc.id);
-    if (idx !== -1) {
-      results[idx] = { id: fallbackDoc.id, title: fallbackDoc.title, path: filePath, bytes: fallbackDoc.content.length, source: "verified_spec" };
-    } else {
-      results.push({ id: fallbackDoc.id, title: fallbackDoc.title, path: filePath, bytes: fallbackDoc.content.length, source: "verified_spec" });
-    }
-  }
-
-  if (!silent) {
-    console.log(`\n  [OK] Corpus ready: ${results.length} documents (${networkCount} fetched, ${cachedCount} cached, ${results.length - networkCount - cachedCount} fallback).\n`);
-  }
-  return results;
-}
-
-/**
- * Returns the total size of the benchmark corpus cache in bytes,
- * or 0 if the cache directory doesn't exist.
- */
-export async function getCorpusCacheSize() {
-  if (!existsSync(CORPUS_DIR)) return 0;
-  let totalBytes = 0;
-  try {
-    const files = await readdir(CORPUS_DIR);
-    const stats = await Promise.all(
-      files.map(async (f) => {
-        try {
-          const s = await stat(join(CORPUS_DIR, f));
-          return s.isFile() ? s.size : 0;
-        } catch {
-          return 0;
-        }
-      })
-    );
-    totalBytes = stats.reduce((sum, s) => sum + s, 0);
-  } catch {
-    // Directory read failed
-  }
-  return totalBytes;
-}
-
-/**
- * Deletes the entire benchmark corpus cache directory.
- */
-export async function clearCorpusCache() {
-  if (!existsSync(CORPUS_DIR)) return false;
-  await rm(CORPUS_DIR, { recursive: true, force: true });
-  return true;
-}
-
-if (process.argv[1] && process.argv[1].includes("fetch_real_corpus.js")) {
-  await fetchRealCorpus();
-}

package/mcp-server/benchmarks/gpu_profile_benchmark.js

@@ -1,170 +0,0 @@
-import { updateConfig, getConfig } from "../config/config_manager.js";
-import { getExtractor, embedBatch, resetExtractor } from "../ml/model_manager.js";
-import { GpuMonitor, ExecutionTracer } from "../ml/gpu_monitor.js";
-
-async function runSinglePass(device, modelName, batchSize, totalItems, onProgress) {
-  updateConfig({ executionDevice: device, embeddingModel: modelName, batchSize });
-  resetExtractor();
-
-  await getExtractor(modelName);
-
-  const allTexts = Array.from({ length: totalItems }, (_, i) =>
-    `High throughput parallel matrix multiplication on GPU using DirectML execution provider. Micro-chunk block #${i + 1} for dense vector embedding computation.`
-  );
-
-  const isGpu = device !== "cpu";
-  const monitor = isGpu ? new GpuMonitor(50) : null;
-  if (monitor) monitor.start();
-
-  const start = Date.now();
-  let totalComputed = 0;
-
-  for (let i = 0; i < allTexts.length; i += batchSize) {
-    const batch = allTexts.slice(i, i + batchSize);
-    const batchVecs = await embedBatch(batch, false, modelName, null, null, {
-      enableTrace: false,
-      enableMonitor: false,
-    });
-    totalComputed += batchVecs.length;
-    if (onProgress) onProgress({ current: totalComputed, total: totalItems, device });
-  }
-
-  const duration = Date.now() - start;
-  const gpuStats = monitor ? monitor.stop() : null;
-
-  global.gc?.({ type: "major" });
-
-  return {
-    device,
-    totalComputed,
-    durationMs: duration,
-    throughput: totalComputed / (duration / 1000),
-    avgMsPerItem: duration / totalComputed,
-    gpuStats,
-  };
-}
-
-export async function runCpuVsGpuComparison(options = {}) {
-  const modelName = options.modelName || getConfig().embeddingModel || "Xenova/bge-m3";
-  const batchSize = options.batchSize || 32;
-  const totalItems = options.totalItems || 512;
-
-  console.log(`\n===============================================================`);
-  console.log(`  CPU vs GPU INFERENCE COMPARISON BENCHMARK`);
-  console.log(`===============================================================`);
-  console.log(`  Model:       ${modelName}`);
-  console.log(`  Batch Size:  ${batchSize}`);
-  console.log(`  Items:       ${totalItems}`);
-  console.log(`  GC Exposed:  ${typeof global.gc === "function" ? "YES" : "NO"}`);
-  console.log(`===============================================================\n`);
-
-  console.log(`  [1/2] Running CPU pass...`);
-  const cpuResult = await runSinglePass("cpu", modelName, batchSize, totalItems, options.onProgress);
-  console.log(`         CPU: ${cpuResult.throughput.toFixed(1)} emb/s, ${cpuResult.avgMsPerItem.toFixed(2)} ms/item, ${cpuResult.durationMs}ms total\n`);
-
-  console.log(`  [2/2] Running GPU pass (DirectML)...`);
-  const gpuResult = await runSinglePass("webgpu", modelName, batchSize, totalItems, options.onProgress);
-  console.log(`         GPU: ${gpuResult.throughput.toFixed(1)} emb/s, ${gpuResult.avgMsPerItem.toFixed(2)} ms/item, ${gpuResult.durationMs}ms total\n`);
-
-  const speedup = cpuResult.durationMs / gpuResult.durationMs;
-  const gpuPeak = gpuResult.gpuStats ? gpuResult.gpuStats.peak : null;
-  const gpuAvg = gpuResult.gpuStats ? gpuResult.gpuStats.avg : null;
-
-  console.log(`===============================================================`);
-  console.log(`  COMPARISON RESULTS`);
-  console.log(`===============================================================`);
-  console.log(`  Metric                 CPU              GPU`);
-  console.log(`  ─────────────────────────────────────────────────────────`);
-  console.log(`  Duration               ${String(cpuResult.durationMs + "ms").padEnd(17)}${gpuResult.durationMs}ms`);
-  console.log(`  Throughput             ${String(cpuResult.throughput.toFixed(1) + " emb/s").padEnd(17)}${gpuResult.throughput.toFixed(1)} emb/s`);
-  console.log(`  Avg per item           ${String(cpuResult.avgMsPerItem.toFixed(2) + " ms").padEnd(17)}${gpuResult.avgMsPerItem.toFixed(2)} ms`);
-  if (gpuPeak !== null) {
-    console.log(`  GPU Peak               -                ${gpuPeak}%`);
-    console.log(`  GPU Average            -                ${gpuAvg}%`);
-  }
-  console.log(`  ─────────────────────────────────────────────────────────`);
-  console.log(`  Speedup:               ${speedup.toFixed(2)}x ${speedup > 1 ? "(GPU faster)" : speedup < 1 ? "(CPU faster)" : "(equal)"}`);
-  console.log(`===============================================================\n`);
-
-  return { cpu: cpuResult, gpu: gpuResult, speedup };
-}
-
-export async function runGpuProfileBenchmark(options = {}) {
-  const modelName = options.modelName || "Xenova/bge-small-en-v1.5";
-  const batchSize = options.batchSize || 256;
-  const totalItems = options.totalItems || 1024;
-  const minGpuThreshold = options.minGpuThreshold || 25;
-
-  console.log(`\n===============================================================`);
-  console.log(` ⚡ GPU HARDWARE INFERENCE & BOTTLENECK PROFILER BENCHMARK`);
-  console.log(`===============================================================`);
-  console.log(` Target Model:          ${modelName}`);
-  console.log(` Batch Size:            ${batchSize} items/batch`);
-  console.log(` Total Items:           ${totalItems}`);
-  console.log(`===============================================================\n`);
-
-  updateConfig({ executionDevice: "webgpu", embeddingModel: modelName, batchSize });
-  resetExtractor();
-
-  console.log("1. Initializing GPU DirectML Engine...");
-  await getExtractor(modelName);
-  console.log("   [OK] Engine Loaded & Initialized on GPU (DirectML)\n");
-
-  console.log("2. Sampling GPU Utilization & Profiling Operations...");
-  const allTexts = Array.from({ length: totalItems }, (_, i) =>
-    `High throughput parallel matrix multiplication on GPU using DirectML execution provider. Micro-chunk block #${i + 1} for dense vector embedding computation.`
-  );
-
-  const monitor = new GpuMonitor(30);
-  monitor.start();
-
-  const tracer = new ExecutionTracer(`GPU Execution (${totalItems} items, batch ${batchSize})`);
-  const start = Date.now();
-  let totalComputed = 0;
-
-  for (let i = 0; i < allTexts.length; i += batchSize) {
-    const batch = allTexts.slice(i, i + batchSize);
-
-    tracer.startStage(`Batch ${i / batchSize + 1} Preprocessing (CPU)`, "CPU");
-    const formatted = batch.map((t) => t.trim());
-
-    tracer.startStage(`Batch ${i / batchSize + 1} Tensor Inference (GPU)`, "GPU");
-    const batchVecs = await embedBatch(formatted, false, modelName, null, null, {
-      enableTrace: false,
-      enableMonitor: false,
-    });
-    totalComputed += batchVecs.length;
-  }
-  tracer.endStage();
-
-  const duration = Date.now() - start;
-  const gpuStats = monitor.stop();
-
-  const summary = tracer.printTraceReport(gpuStats, minGpuThreshold);
-
-  console.log(`=== BENCHMARK SUMMARY & METRICS ===`);
-  console.log(` - Total Throughput:     ${(totalComputed / (duration / 1000)).toFixed(1)} embeddings / second`);
-  console.log(` - Average Per-Item:     ${(duration / totalComputed).toFixed(2)} ms / item`);
-  console.log(` - Peak GPU Load:        ${gpuStats.peak}%`);
-  console.log(` - Average GPU Load:     ${gpuStats.avg}%`);
-  console.log(` - GPU Time Share:       ${summary.gpuMs.toFixed(1)}ms (${summary.gpuPct}% of total time)`);
-  console.log(` - CPU Time Share:       ${summary.cpuMs.toFixed(1)}ms (${summary.cpuPct}% of total time)`);
-  console.log(`===============================================================\n`);
-
-  return {
-    totalComputed,
-    durationMs: duration,
-    throughput: totalComputed / (duration / 1000),
-    gpuStats,
-    summary,
-  };
-}
-
-if (process.argv[1]?.includes("gpu_profile_benchmark.js")) {
-  const args = process.argv.slice(2);
-  const fn = args.includes("--compare") ? runCpuVsGpuComparison : runGpuProfileBenchmark;
-  fn().catch((err) => {
-    console.error("\n❌ BENCHMARK ABORTED:", err.message);
-    process.exit(1);
-  });
-}