@deepsweet/mdn 0.1.0

package/LICENSE.md

@@ -0,0 +1,21 @@
+# The MIT License (MIT)
+
+Copyright (c) 2026–present Kir Belevich
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/README.md

@@ -0,0 +1,68 @@
+Offline-first [MDN Web Docs](https://developer.mozilla.org/) RAG-MCP server ready for semantic search with hybrid vector (1024-d) and full‑text (BM25) retrieval.
+
+## Example
+
+![example screenshot](example.webp)
+
+## Content
+
+The dataset covers the core MDN documentation sections, including:
+
+- Web API
+- JavaScript
+- HTML
+- CSS
+- SVG
+- HTTP
+
+See [dataset repo](https://huggingface.co/datasets/deepsweet/mdn) on HuggigFace for more details.
+
+## Usage
+
+### 1. Download dataset and embedding model
+
+```sh
+npx @deepsweet/mdn@latest download
+```
+
+Both [dataset](https://huggingface.co/datasets/deepsweet/mdn) (\~260 MB) and the [embedding model GGUF file](https://huggingface.co/deepsweet/bge-m3-GGUF-Q4_K_M) (\~438 MB) will be downloaded directly from HugginFace and stored in its default cache location (typically `~/.cache/huggingface/`), just like the `hf download` command does.
+
+### 2. Setup RAG-MCP server
+
+```json
+{
+  "mcpServers": {
+    "mdn": {
+      "command": "npx",
+      "args": [
+        "@deepsweet/mdn",
+        "server"
+      ],
+      "env": {}
+    }
+  }
+}
+```
+
+The `stdio` server will spawn [llama.cpp](https://github.com/ggml-org/llama.cpp) under the hood, load the embedding model (~655 MB RAM/VRAM), and query the dataset – all on demand.
+
+## Settings
+
+| Env variable               | Default value                                                   | Description                                                                                                       |
+|----------------------------|-----------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
+| `MDN_DATASET_PATH`         | HuggingFace cache                                               | Custom dataset directory path                                                                                     |
+| `MDN_DATASET_LOCALE`       | `en-us`                                                         | Dataset language, currently `en-us` only                                                                          |
+| `MDN_MODEL_PATH`           | HuggingFace cache                                               | Custom model file path                                                                                            |
+| `MDN_MODEL_TTL`            | `1800`                                                          | For how long llama.cpp with embedding model should be kept loaded in memory, in seconds; `0` to prevent unloading |
+| `MDN_QUERY_DESCRIPTION`    | `Natural language query for hybrid vector and full-text search` | Custom search query description in case your LLM does a poor job asking the MCP tool                              |
+| `MDN_SEARCH_RESULTS_LIMIT` | `3`                                                             | Total search results limit                                                                                        |
+
+## To do
+
+- [ ] figure out a better query description so that LLM doesn't over-generate keywords
+- [ ] add more dataset [translations](https://github.com/mdn/translated-content/tree/main/files/)
+- [ ] automatically update and upload the dataset artifacts monthly with GitHub Actions
+
+## License
+
+The RAG-MCP server itself and the processing scripts are available under MIT.

package/dist/index.js

@@ -0,0 +1,193 @@
+#!/usr/bin/env node
+
+// src/env.ts
+import { z } from "zod";
+var env = z.object({
+  MDN_DATASET_PATH: z.string().optional(),
+  MDN_DATASET_LOCALE: z.enum(["en-us"]).default("en-us"),
+  MDN_MODEL_PATH: z.string().optional(),
+  MDN_MODEL_TTL: z.number().default(1800),
+  MDN_QUERY_DESCRIPTION: z.string().default("Natural language query for hybrid vector and full-text search"),
+  MDN_SEARCH_RESULTS_LIMIT: z.coerce.number().default(3)
+}).parse(process.env);
+
+// src/huggingface.ts
+import fs from "node:fs/promises";
+import path from "path";
+import { getHFHubCachePath, getRepoFolderName, scanCachedRepo, snapshotDownload } from "@huggingface/hub";
+
+// src/utils.ts
+var getTableName = (locale) => {
+  return `mdn-${locale}`;
+};
+
+// src/huggingface.ts
+var DATASET_REPO = "deepsweet/mdn";
+var MODEL_REPO = "deepsweet/bge-m3-GGUF-Q4_K_M";
+var MODEL_FILE = "bge-m3-GGUF-Q4_K_M.gguf";
+var replaceSymlinksWithHardlinks = async (dir) => {
+  const entries = await fs.readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isSymbolicLink()) {
+      const target = await fs.readlink(fullPath);
+      const sourcePath = path.resolve(dir, target);
+      await fs.unlink(fullPath);
+      await fs.link(sourcePath, fullPath);
+      continue;
+    }
+    if (entry.isDirectory()) {
+      await replaceSymlinksWithHardlinks(fullPath);
+    }
+  }
+};
+var getLatestCachedRepoRevision = async (name, type) => {
+  const cachePath = getHFHubCachePath();
+  const repoFolderName = getRepoFolderName({ name, type });
+  const repoPath = path.join(cachePath, repoFolderName);
+  const repo = await scanCachedRepo(repoPath);
+  if (repo.revisions.length === 0) {
+    throw new Error("Unable to get model path, it needs to be downloaded first");
+  }
+  const latestRevision = repo.revisions.reduce((latest, current) => {
+    if (current.lastModifiedAt > latest.lastModifiedAt) {
+      return current;
+    }
+    return latest;
+  });
+  return latestRevision.path;
+};
+var downloadDataset = async (locale) => {
+  const tableName = getTableName(locale);
+  const dirPath = await snapshotDownload({
+    repo: `datasets/${DATASET_REPO}`,
+    path: `data/${tableName}.lance`
+  });
+  const dataPath = path.join(dirPath, "data");
+  await replaceSymlinksWithHardlinks(dataPath);
+};
+var getDatasetPath = async () => {
+  if (process.env.MDN_DATASET_PATH != null) {
+    return process.env.MDN_DATASET_PATH;
+  }
+  const latestRevisionPath = await getLatestCachedRepoRevision(DATASET_REPO, "dataset");
+  const datasetPath = path.join(latestRevisionPath, "data");
+  return datasetPath;
+};
+var downloadModel = async () => {
+  await snapshotDownload({
+    repo: MODEL_REPO
+  });
+};
+var getModelPath = async () => {
+  if (process.env.MDN_MODEL_PATH != null) {
+    return process.env.MDN_MODEL_PATH;
+  }
+  const latestRevisionPath = await getLatestCachedRepoRevision(MODEL_REPO, "model");
+  const modelPath = path.join(latestRevisionPath, MODEL_FILE);
+  return modelPath;
+};
+
+// src/server.ts
+import lancedb2 from "@lancedb/lancedb";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { z as z2 } from "zod";
+
+// src/llama.ts
+import { getLlama } from "node-llama-cpp";
+var MAX_TOKENS = 8192;
+var getLlamaContext = async (modelPath) => {
+  const llama = await getLlama();
+  const model = await llama.loadModel({ modelPath });
+  const context = await model.createEmbeddingContext({
+    contextSize: MAX_TOKENS,
+    batchSize: MAX_TOKENS,
+    threads: 0
+  });
+  return context;
+};
+
+// src/query.ts
+import lancedb from "@lancedb/lancedb";
+
+// src/vectorize.ts
+var vectorize = async (context, text) => {
+  const embedding = await context.getEmbeddingFor(text);
+  const vector = embedding.vector;
+  return vector;
+};
+
+// src/query.ts
+var queryHybrid = async (llamaContext, table, reranker, text) => {
+  const vector = await vectorize(llamaContext, text);
+  const results = await table.query().nearestTo(vector).fullTextSearch(text).rerank(reranker).limit(env.MDN_SEARCH_RESULTS_LIMIT).toArray();
+  return results;
+};
+var createReranker = async () => {
+  const reranker = await lancedb.rerankers.RRFReranker.create();
+  return reranker;
+};
+
+// package.json
+var name = "@deepsweet/mdn";
+var version = "0.1.0";
+
+// src/server.ts
+var startMcpServer = async () => {
+  const datasetPath = await getDatasetPath();
+  const db = await lancedb2.connect(datasetPath);
+  const reranker = await createReranker();
+  const server = new McpServer({ name, version });
+  const tableName = getTableName(env.MDN_DATASET_LOCALE);
+  const table = await db.openTable(tableName);
+  const modelPath = await getModelPath();
+  const llamaTtl = env.MDN_MODEL_TTL * 1000;
+  let llamaContext = null;
+  let llamaTimeout = null;
+  server.registerTool("MDN", {
+    description: "Reference documentation for Web API, JavaScript, HTML, CSS, SVG and HTTP",
+    inputSchema: z2.object({
+      query: z2.string().describe(env.MDN_QUERY_DESCRIPTION)
+    })
+  }, async ({ query }) => {
+    llamaContext ??= await getLlamaContext(modelPath);
+    if (env.MDN_MODEL_TTL > 0) {
+      if (llamaTimeout !== null) {
+        clearTimeout(llamaTimeout);
+      }
+      llamaTimeout = setTimeout(() => {
+        llamaTimeout = null;
+        llamaContext?.dispose().catch(console.error);
+        llamaContext = null;
+      }, llamaTtl);
+    }
+    const results = await queryHybrid(llamaContext, table, reranker, query);
+    return {
+      content: results.map((result) => ({
+        type: "text",
+        text: result.text
+      }))
+    };
+  });
+  const transport = new StdioServerTransport;
+  await server.connect(transport);
+};
+
+// src/index.ts
+switch (process.argv[2]) {
+  case "download": {
+    const locale = process.argv[3] ?? env.MDN_DATASET_LOCALE;
+    await downloadDataset(locale);
+    await downloadModel();
+    break;
+  }
+  case "server": {
+    await startMcpServer();
+    break;
+  }
+  default: {
+    console.error('Unknown or missing command, use "download" or "server"');
+    process.exit(1);
+  }
+}

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@deepsweet/mdn",
+  "version": "0.1.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "bin": "dist/index.js",
+  "main": "dist/index.js",
+  "files": [
+    "dist/"
+  ],
+  "dependencies": {
+    "@huggingface/hub": "^2.11.0",
+    "@lancedb/lancedb": "0.27.2-beta.1",
+    "@modelcontextprotocol/sdk": "^1.29.0",
+    "apache-arrow": "18.1.0",
+    "node-llama-cpp": "^3.18.1",
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "@huggingface/tokenizers": "^0.1.3",
+    "@stylistic/eslint-plugin": "^5.10.0",
+    "@types/bun": "^1.3.11",
+    "dedent": "^1.7.2",
+    "eslint": "^10.1.0",
+    "eslint-plugin-import": "^2.32.0",
+    "eslint-plugin-perfectionist": "^5.7.0",
+    "globals": "^17.4.0",
+    "gray-matter": "^4.0.3",
+    "marked": "^17.0.5",
+    "rimraf": "^6.1.3",
+    "typescript": "^6.0.2",
+    "typescript-eslint": "^8.58.0"
+  },
+  "scripts": {
+    "chunk": "bun scripts/chunk.ts",
+    "ingest": "bun scripts/ingest.ts",
+    "query": "bun scripts/query.ts",
+    "check": "tsc --noEmit && eslint --cache scripts/ src/",
+    "dist": "bun build --format esm --target node --packages external --outdir dist/ src/index.ts"
+  },
+  "license": "MIT"
+}