inkdex 0.0.1

package/dist/ingest/index-docs.js

@@ -0,0 +1,78 @@
+import { createHash } from "node:crypto";
+import { glob, readFile } from "node:fs/promises";
+import { relative } from "node:path";
+import { logger } from "../logger.js";
+import { getAllDocumentHashes, insertChunk, removeDocument, runInTransaction, setDocumentHash, } from "../store/db.js";
+import { chunkMarkdown } from "./chunker.js";
+const MAX_CHUNK_FILL = 0.8;
+async function findMarkdownFiles(docsPath) {
+    const files = [];
+    for await (const entry of glob("**/*.md", { cwd: docsPath })) {
+        files.push(`${docsPath}/${entry}`);
+    }
+    return files.sort();
+}
+function hashContent(content) {
+    return createHash("sha256").update(content).digest("hex");
+}
+export async function indexDocs(embedder, docsPath) {
+    const files = await findMarkdownFiles(docsPath);
+    if (files.length === 0) {
+        logger.warn({ path: docsPath }, "No markdown files found");
+        return;
+    }
+    const fileContents = new Map();
+    for (const file of files) {
+        const key = relative(docsPath, file);
+        const content = await readFile(file, "utf-8");
+        fileContents.set(key, content);
+    }
+    const storedHashes = getAllDocumentHashes();
+    const changedKeys = [];
+    for (const [key, content] of fileContents) {
+        if (storedHashes[key] !== hashContent(content)) {
+            changedKeys.push(key);
+        }
+    }
+    const removedKeys = [];
+    for (const key of Object.keys(storedHashes)) {
+        if (!fileContents.has(key)) {
+            removedKeys.push(key);
+        }
+    }
+    if (changedKeys.length === 0 && removedKeys.length === 0) {
+        logger.info({ files: files.length }, "Index up to date");
+        return;
+    }
+    const start = performance.now();
+    logger.info({ changed: changedKeys.length, removed: removedKeys.length }, "Indexing changed files");
+    if (removedKeys.length > 0) {
+        runInTransaction(() => {
+            for (const key of removedKeys) {
+                removeDocument(key);
+            }
+        });
+    }
+    const chunkOptions = {
+        maxTokens: Math.floor(embedder.maxTokens * MAX_CHUNK_FILL),
+        countTokens: (text) => embedder.tokenize(text).length,
+    };
+    let totalChunks = 0;
+    for (const key of changedKeys) {
+        const content = fileContents.get(key);
+        const chunks = chunkMarkdown(content, key, chunkOptions);
+        logger.debug({ path: key, chunks: chunks.length }, "Embedding chunks");
+        const embeddings = await embedder.embedBatch(chunks.map((c) => c.text));
+        runInTransaction(() => {
+            removeDocument(key);
+            setDocumentHash(key, hashContent(content));
+            for (let i = 0; i < chunks.length; i++) {
+                const chunk = chunks[i];
+                insertChunk(chunk.path, chunk.fileHeading, chunk.heading, chunk.text, chunk.metadata, embeddings[i]);
+            }
+        });
+        totalChunks += chunks.length;
+    }
+    const duration = ((performance.now() - start) / 1000).toFixed(1);
+    logger.info({ duration: `${duration}s`, chunks: totalChunks }, "Indexing complete");
+}

package/dist/logger.d.ts

@@ -0,0 +1,6 @@
+export declare const logger: {
+    debug: (msgOrObj: string | Record<string, unknown>, msg?: string) => void;
+    info: (msgOrObj: string | Record<string, unknown>, msg?: string) => void;
+    warn: (msgOrObj: string | Record<string, unknown>, msg?: string) => void;
+    error: (msgOrObj: string | Record<string, unknown>, msg?: string) => void;
+};

package/dist/logger.js

@@ -0,0 +1,28 @@
+const level = process.env.LOG_LEVEL ?? "info";
+const levels = {
+    debug: 0,
+    info: 1,
+    warn: 2,
+    error: 3,
+};
+const threshold = levels[level] ?? 1;
+// All levels go to stderr to keep stdout free for the MCP stdio transport
+function log(lvl, msg) {
+    if ((levels[lvl] ?? 0) >= threshold) {
+        console.error(`[${lvl.toUpperCase()}] ${msg}`);
+    }
+}
+export const logger = {
+    debug: (msgOrObj, msg) => log("debug", formatMsg(msgOrObj, msg)),
+    info: (msgOrObj, msg) => log("info", formatMsg(msgOrObj, msg)),
+    warn: (msgOrObj, msg) => log("warn", formatMsg(msgOrObj, msg)),
+    error: (msgOrObj, msg) => log("error", formatMsg(msgOrObj, msg)),
+};
+function formatMsg(msgOrObj, msg) {
+    if (typeof msgOrObj === "string")
+        return msgOrObj;
+    const data = Object.entries(msgOrObj)
+        .map(([k, v]) => `${k}=${v}`)
+        .join(" ");
+    return msg ? `${msg} ${data}` : data;
+}

package/dist/search/search.d.ts

@@ -0,0 +1,7 @@
+import type { Embedder } from "../embedder/embedder.js";
+import type { ChunkRow, SearchResult } from "../types.js";
+/** @package */
+export declare function cosineSimilarity(a: number[], b: number[]): number;
+/** @package */
+export declare function rankChunksHybrid(chunks: ChunkRow[], queryEmbedding: number[], ftsRankedIds: number[], limit: number): SearchResult[];
+export declare function search(embedder: Embedder, query: string, limit: number): Promise<SearchResult[]>;

package/dist/search/search.js

@@ -0,0 +1,70 @@
+import { getAllChunks, searchFts } from "../store/db.js";
+/** @package */
+export function cosineSimilarity(a, b) {
+    let dot = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+const RRF_K = 60;
+/** @package */
+export function rankChunksHybrid(chunks, queryEmbedding, ftsRankedIds, limit) {
+    const vectorRanked = chunks
+        .map((chunk) => ({
+        chunk,
+        similarity: cosineSimilarity(queryEmbedding, chunk.embedding),
+    }))
+        .sort((a, b) => b.similarity - a.similarity);
+    const vectorRankMap = new Map();
+    for (let i = 0; i < vectorRanked.length; i++) {
+        vectorRankMap.set(vectorRanked[i].chunk.id, i + 1);
+    }
+    const bm25RankMap = new Map();
+    for (let i = 0; i < ftsRankedIds.length; i++) {
+        bm25RankMap.set(ftsRankedIds[i], i + 1);
+    }
+    const chunkById = new Map();
+    for (const chunk of chunks) {
+        chunkById.set(chunk.id, chunk);
+    }
+    const allIds = new Set([
+        ...vectorRankMap.keys(),
+        ...bm25RankMap.keys(),
+    ]);
+    const scored = [];
+    for (const id of allIds) {
+        const chunk = chunkById.get(id);
+        if (!chunk)
+            continue;
+        const vectorRank = vectorRankMap.get(id);
+        const bm25Rank = bm25RankMap.get(id);
+        let score = 0;
+        if (vectorRank !== undefined)
+            score += 1 / (RRF_K + vectorRank);
+        if (bm25Rank !== undefined)
+            score += 1 / (RRF_K + bm25Rank);
+        scored.push({ chunk, score });
+    }
+    return scored
+        .sort((a, b) => b.score - a.score)
+        .slice(0, limit)
+        .map(({ chunk, score }) => ({
+        path: chunk.path,
+        fileHeading: chunk.fileHeading,
+        heading: chunk.heading,
+        text: chunk.text,
+        metadata: chunk.metadata,
+        score,
+    }));
+}
+export async function search(embedder, query, limit) {
+    const queryEmbedding = await embedder.embed(query);
+    const chunks = getAllChunks();
+    const ftsRankedIds = searchFts(query, chunks.length);
+    return rankChunksHybrid(chunks, queryEmbedding, ftsRankedIds, limit);
+}

package/dist/server.d.ts

@@ -0,0 +1,2 @@
+import type { Embedder } from "./embedder/embedder.js";
+export declare function startServer(embedder: Embedder): Promise<void>;

package/dist/server.js

@@ -0,0 +1,66 @@
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { CallToolRequestSchema, ListToolsRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
+import { logger } from "./logger.js";
+import { search } from "./search/search.js";
+import { getChunkCount } from "./store/db.js";
+import { getVersion } from "./version.js";
+async function createServer(embedder) {
+    const server = new Server({
+        name: "inkdex",
+        version: getVersion(),
+    }, {
+        capabilities: {
+            tools: {},
+        },
+    });
+    server.setRequestHandler(ListToolsRequestSchema, async () => {
+        return {
+            tools: [
+                {
+                    name: "search_docs",
+                    description: "Search markdown documentation for relevant information. Returns chunks of content that match the query semantically.",
+                    inputSchema: {
+                        type: "object",
+                        properties: {
+                            query: {
+                                type: "string",
+                                description: "Search query - natural language question or keywords",
+                            },
+                            limit: {
+                                type: "number",
+                                description: "Maximum number of results to return (1-20)",
+                                default: 5,
+                                minimum: 1,
+                                maximum: 20,
+                            },
+                        },
+                        required: ["query"],
+                    },
+                },
+            ],
+        };
+    });
+    server.setRequestHandler(CallToolRequestSchema, async (request) => {
+        if (request.params.name !== "search_docs") {
+            throw new Error(`Unknown tool: ${request.params.name}`);
+        }
+        const query = String(request.params.arguments?.query || "");
+        const limit = Math.min(Math.max(Number(request.params.arguments?.limit) || 5, 1), 20);
+        logger.debug({ query, limit }, "Searching docs");
+        const results = await search(embedder, query, limit);
+        const text = results
+            .map((r) => `## ${r.fileHeading} > ${r.heading}\n_Source: ${r.path} (score: ${r.score.toFixed(3)})_\n\n${r.text}`)
+            .join("\n\n---\n\n");
+        return {
+            content: [{ type: "text", text: text || "No results found." }],
+        };
+    });
+    return server;
+}
+export async function startServer(embedder) {
+    const server = await createServer(embedder);
+    const transport = new StdioServerTransport();
+    await server.connect(transport);
+    logger.info({ version: getVersion(), chunks: getChunkCount() }, "Server started");
+}

package/dist/store/db.d.ts

@@ -0,0 +1,13 @@
+import type { ChunkRow } from "../types.js";
+/** @package */
+export declare function dbPath(docsPath: string): string;
+export declare function openDb(docsPath: string): void;
+export declare function closeDb(): void;
+export declare function getAllDocumentHashes(): Record<string, string>;
+export declare function setDocumentHash(path: string, hash: string): void;
+export declare function removeDocument(path: string): void;
+export declare function insertChunk(documentPath: string, fileHeading: string, heading: string, text: string, metadata: Record<string, unknown>, embedding: number[]): void;
+export declare function getAllChunks(): ChunkRow[];
+export declare function getChunkCount(): number;
+export declare function searchFts(query: string, limit: number): number[];
+export declare function runInTransaction(fn: () => void): void;

package/dist/store/db.js

@@ -0,0 +1,149 @@
+import { createHash } from "node:crypto";
+import { mkdirSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+const STORE_DIR = join(homedir(), ".inkdex");
+/** @package */
+export function dbPath(docsPath) {
+    const hash = createHash("sha256").update(docsPath).digest("hex").slice(0, 12);
+    return join(STORE_DIR, `${hash}.db`);
+}
+const SCHEMA_VERSION = 2;
+const CHUNK_COLUMNS = "id, document_path, file_heading, heading, text, metadata, embedding";
+let db;
+let stmts;
+function createSchema() {
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS documents (
+      path TEXT PRIMARY KEY,
+      hash TEXT NOT NULL
+    );
+    CREATE TABLE IF NOT EXISTS chunks (
+      id INTEGER PRIMARY KEY,
+      document_path TEXT NOT NULL REFERENCES documents(path),
+      file_heading TEXT NOT NULL,
+      heading TEXT NOT NULL,
+      text TEXT NOT NULL,
+      metadata TEXT NOT NULL,
+      embedding BLOB NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_chunks_document ON chunks(document_path);
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts
+      USING fts5(text, content=chunks, content_rowid=id);
+
+    CREATE TRIGGER IF NOT EXISTS chunks_fts_insert AFTER INSERT ON chunks BEGIN
+      INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text);
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS chunks_fts_delete AFTER DELETE ON chunks BEGIN
+      INSERT INTO chunks_fts(chunks_fts, rowid, text)
+        VALUES('delete', old.id, old.text);
+    END;
+  `);
+    db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`);
+}
+function prepareStatements() {
+    stmts = {
+        getAllDocs: db.prepare("SELECT path, hash FROM documents"),
+        upsertDoc: db.prepare("INSERT OR REPLACE INTO documents (path, hash) VALUES (?, ?)"),
+        deleteChunksByDoc: db.prepare("DELETE FROM chunks WHERE document_path = ?"),
+        deleteDoc: db.prepare("DELETE FROM documents WHERE path = ?"),
+        insertChunk: db.prepare("INSERT INTO chunks (document_path, file_heading, heading, text, metadata, embedding) VALUES (?, ?, ?, ?, ?, ?)"),
+        getAllChunks: db.prepare(`SELECT ${CHUNK_COLUMNS} FROM chunks`),
+        countChunks: db.prepare("SELECT COUNT(*) as count FROM chunks"),
+        searchFts: db.prepare("SELECT rowid FROM chunks_fts WHERE chunks_fts MATCH ? ORDER BY bm25(chunks_fts) LIMIT ?"),
+    };
+}
+export function openDb(docsPath) {
+    mkdirSync(STORE_DIR, { recursive: true });
+    db = new DatabaseSync(dbPath(docsPath));
+    db.exec("PRAGMA journal_mode = WAL");
+    db.exec("PRAGMA foreign_keys = ON");
+    const { user_version } = db.prepare("PRAGMA user_version").get();
+    if (user_version !== SCHEMA_VERSION) {
+        db.exec("DROP TABLE IF EXISTS chunks");
+        db.exec("DROP TABLE IF EXISTS documents");
+        createSchema();
+    }
+    else {
+        createSchema();
+    }
+    prepareStatements();
+}
+export function closeDb() {
+    db?.close();
+}
+export function getAllDocumentHashes() {
+    const rows = stmts.getAllDocs.all();
+    const result = {};
+    for (const row of rows) {
+        result[row.path] = row.hash;
+    }
+    return result;
+}
+export function setDocumentHash(path, hash) {
+    stmts.upsertDoc.run(path, hash);
+}
+export function removeDocument(path) {
+    stmts.deleteChunksByDoc.run(path);
+    stmts.deleteDoc.run(path);
+}
+function embeddingToBlob(embedding) {
+    return Buffer.from(new Float32Array(embedding).buffer);
+}
+function blobToEmbedding(blob) {
+    return Array.from(new Float32Array(new Uint8Array(blob).buffer));
+}
+function toChunkRow(row) {
+    return {
+        id: row.id,
+        path: row.document_path,
+        fileHeading: row.file_heading,
+        heading: row.heading,
+        text: row.text,
+        metadata: JSON.parse(row.metadata),
+        embedding: blobToEmbedding(row.embedding),
+    };
+}
+export function insertChunk(documentPath, fileHeading, heading, text, metadata, embedding) {
+    stmts.insertChunk.run(documentPath, fileHeading, heading, text, JSON.stringify(metadata), embeddingToBlob(embedding));
+}
+export function getAllChunks() {
+    const rows = stmts.getAllChunks.all();
+    return rows.map(toChunkRow);
+}
+export function getChunkCount() {
+    const row = stmts.countChunks.get();
+    return row.count;
+}
+export function searchFts(query, limit) {
+    // Sanitize: split into words, quote each to escape FTS5 operators
+    const terms = query
+        .split(/\s+/)
+        .filter((t) => t.length > 0)
+        .map((t) => `"${t.replace(/"/g, '""')}"`)
+        .join(" ");
+    if (terms.length === 0)
+        return [];
+    try {
+        const rows = stmts.searchFts.all(terms, limit);
+        return rows.map((r) => r.rowid);
+    }
+    catch {
+        // FTS5 MATCH can fail on edge-case inputs; fall back to empty
+        return [];
+    }
+}
+export function runInTransaction(fn) {
+    db.exec("BEGIN");
+    try {
+        fn();
+        db.exec("COMMIT");
+    }
+    catch (error) {
+        db.exec("ROLLBACK");
+        throw error;
+    }
+}

package/dist/types.d.ts

@@ -0,0 +1,14 @@
+export interface BaseChunk {
+    path: string;
+    fileHeading: string;
+    heading: string;
+    text: string;
+    metadata: Record<string, unknown>;
+}
+export interface ChunkRow extends BaseChunk {
+    id: number;
+    embedding: number[];
+}
+export interface SearchResult extends Omit<ChunkRow, "id" | "embedding"> {
+    score: number;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/dist/version.d.ts

@@ -0,0 +1 @@
+export declare function getVersion(): string;

package/dist/version.js

@@ -0,0 +1,13 @@
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+export function getVersion() {
+    try {
+        const packageJson = JSON.parse(readFileSync(join(__dirname, "..", "package.json"), "utf-8"));
+        return packageJson.version || "unknown";
+    }
+    catch {
+        return "unknown";
+    }
+}

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "inkdex",
+  "version": "0.0.1",
+  "description": "MCP server that makes your markdown docs searchable",
+  "type": "module",
+  "main": "dist/cli.js",
+  "bin": {
+    "inkdex": "dist/cli.js"
+  },
+  "engines": {
+    "node": ">=22.5.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx src/cli.ts",
+    "start": "node dist/cli.js",
+    "test": "npm run test:unit && npm run test:integration",
+    "test:unit": "tsx --test test/unit/**/*.test.ts",
+    "test:integration": "tsx --test --test-concurrency=1 test/integration/**/*.test.ts",
+    "check": "biome check .",
+    "format": "biome format --write ."
+  },
+  "keywords": [
+    "mcp",
+    "mcp-server",
+    "model-context-protocol",
+    "documentation",
+    "semantic-search",
+    "embeddings",
+    "markdown",
+    "rag"
+  ],
+  "author": "Anton Lundén",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@huggingface/transformers": "^3.8.1",
+    "@modelcontextprotocol/sdk": "^1.25.3",
+    "gray-matter": "^4.0.3"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^2.3.14",
+    "@types/node": "^25.1.0",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3"
+  }
+}

package/release.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "${1:-}" =~ ^(-h|--help)$ ]] || [[ -z "${1:-}" ]]; then
+  echo "Usage: ./release.sh <version>"
+  echo "  version: X.Y.Z (e.g., 0.1.0)"
+  exit 0
+fi
+
+VERSION="$1"
+
+if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+  echo "Error: Version must be in format X.Y.Z (e.g., 0.1.0)"
+  exit 1
+fi
+
+echo "Releasing v$VERSION"
+
+npm version "$VERSION" --no-git-tag-version
+npm install
+
+npm audit --audit-level=critical
+npm run check
+npm run test:unit
+npm run test:integration
+
+npm run build
+
+git add package.json package-lock.json
+git commit -m "v$VERSION"
+git tag "v$VERSION"
+
+echo "Done. Push with: git push origin main v$VERSION"

package/src/cli.ts

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+
+import { stat } from "node:fs/promises";
+import { resolve } from "node:path";
+import { Embedder } from "./embedder/embedder.js";
+import { indexDocs } from "./ingest/index-docs.js";
+import { logger } from "./logger.js";
+import { startServer } from "./server.js";
+import { closeDb, openDb } from "./store/db.js";
+
+process.on("uncaughtException", (error) => {
+  logger.error({ error }, "Uncaught exception");
+  process.exit(1);
+});
+
+process.on("unhandledRejection", (reason) => {
+  logger.error({ reason }, "Unhandled rejection");
+  process.exit(1);
+});
+
+async function main(): Promise<void> {
+  const docsPath = process.env.DOCS_PATH;
+  if (!docsPath) {
+    logger.error("DOCS_PATH environment variable is required");
+    process.exit(1);
+  }
+
+  const resolved = resolve(docsPath);
+  const info = await stat(resolved).catch(() => null);
+  if (!info?.isDirectory()) {
+    logger.error({ path: resolved }, "DOCS_PATH is not a directory");
+    process.exit(1);
+  }
+
+  const embedder = await Embedder.load();
+  openDb(resolved);
+  await indexDocs(embedder, resolved);
+  await startServer(embedder);
+}
+
+main().catch((error) => {
+  closeDb();
+  logger.error({ error }, "Failed to start server");
+  process.exit(1);
+});

package/src/embedder/embedder.ts

@@ -0,0 +1,52 @@
+import type { FeatureExtractionPipeline } from "@huggingface/transformers";
+import { pipeline } from "@huggingface/transformers";
+
+const MODEL = "Xenova/all-MiniLM-L6-v2";
+const BATCH_SIZE = 32;
+
+export class Embedder {
+  readonly maxTokens: number;
+  private readonly pipeline: FeatureExtractionPipeline;
+
+  private constructor(pipe: FeatureExtractionPipeline) {
+    this.pipeline = pipe;
+    this.maxTokens = (pipe.tokenizer.model_max_length as number) ?? 256;
+  }
+
+  static async load(): Promise<Embedder> {
+    const pipe = await pipeline<"feature-extraction">(
+      "feature-extraction",
+      MODEL,
+    );
+    return new Embedder(pipe);
+  }
+
+  tokenize(text: string): number[] {
+    return this.pipeline.tokenizer.encode(text);
+  }
+
+  async embed(text: string): Promise<number[]> {
+    const result = await this.pipeline(text, {
+      pooling: "mean",
+      normalize: true,
+    });
+    return (result.tolist() as number[][])[0];
+  }
+
+  async embedBatch(texts: string[]): Promise<number[][]> {
+    if (texts.length === 0) return [];
+
+    const results: number[][] = [];
+
+    for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+      const batch = texts.slice(i, i + BATCH_SIZE);
+      const result = await this.pipeline(batch, {
+        pooling: "mean",
+        normalize: true,
+      });
+      results.push(...(result.tolist() as number[][]));
+    }
+
+    return results;
+  }
+}