inkdex 0.0.1

package/src/store/db.ts

@@ -0,0 +1,217 @@
+import { createHash } from "node:crypto";
+import { mkdirSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import type { StatementSync } from "node:sqlite";
+import { DatabaseSync } from "node:sqlite";
+import type { ChunkRow } from "../types.js";
+
+const STORE_DIR = join(homedir(), ".inkdex");
+
+/** @package */
+export function dbPath(docsPath: string): string {
+  const hash = createHash("sha256").update(docsPath).digest("hex").slice(0, 12);
+  return join(STORE_DIR, `${hash}.db`);
+}
+const SCHEMA_VERSION = 2;
+
+const CHUNK_COLUMNS =
+  "id, document_path, file_heading, heading, text, metadata, embedding";
+
+let db: DatabaseSync;
+
+let stmts: {
+  getAllDocs: StatementSync;
+  upsertDoc: StatementSync;
+  deleteChunksByDoc: StatementSync;
+  deleteDoc: StatementSync;
+  insertChunk: StatementSync;
+  getAllChunks: StatementSync;
+  countChunks: StatementSync;
+  searchFts: StatementSync;
+};
+
+function createSchema(): void {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS documents (
+      path TEXT PRIMARY KEY,
+      hash TEXT NOT NULL
+    );
+    CREATE TABLE IF NOT EXISTS chunks (
+      id INTEGER PRIMARY KEY,
+      document_path TEXT NOT NULL REFERENCES documents(path),
+      file_heading TEXT NOT NULL,
+      heading TEXT NOT NULL,
+      text TEXT NOT NULL,
+      metadata TEXT NOT NULL,
+      embedding BLOB NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_chunks_document ON chunks(document_path);
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts
+      USING fts5(text, content=chunks, content_rowid=id);
+
+    CREATE TRIGGER IF NOT EXISTS chunks_fts_insert AFTER INSERT ON chunks BEGIN
+      INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text);
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS chunks_fts_delete AFTER DELETE ON chunks BEGIN
+      INSERT INTO chunks_fts(chunks_fts, rowid, text)
+        VALUES('delete', old.id, old.text);
+    END;
+  `);
+  db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`);
+}
+
+function prepareStatements(): void {
+  stmts = {
+    getAllDocs: db.prepare("SELECT path, hash FROM documents"),
+    upsertDoc: db.prepare(
+      "INSERT OR REPLACE INTO documents (path, hash) VALUES (?, ?)",
+    ),
+    deleteChunksByDoc: db.prepare("DELETE FROM chunks WHERE document_path = ?"),
+    deleteDoc: db.prepare("DELETE FROM documents WHERE path = ?"),
+    insertChunk: db.prepare(
+      "INSERT INTO chunks (document_path, file_heading, heading, text, metadata, embedding) VALUES (?, ?, ?, ?, ?, ?)",
+    ),
+    getAllChunks: db.prepare(`SELECT ${CHUNK_COLUMNS} FROM chunks`),
+    countChunks: db.prepare("SELECT COUNT(*) as count FROM chunks"),
+    searchFts: db.prepare(
+      "SELECT rowid FROM chunks_fts WHERE chunks_fts MATCH ? ORDER BY bm25(chunks_fts) LIMIT ?",
+    ),
+  };
+}
+
+export function openDb(docsPath: string): void {
+  mkdirSync(STORE_DIR, { recursive: true });
+  db = new DatabaseSync(dbPath(docsPath));
+  db.exec("PRAGMA journal_mode = WAL");
+  db.exec("PRAGMA foreign_keys = ON");
+
+  const { user_version } = db.prepare("PRAGMA user_version").get() as {
+    user_version: number;
+  };
+
+  if (user_version !== SCHEMA_VERSION) {
+    db.exec("DROP TABLE IF EXISTS chunks");
+    db.exec("DROP TABLE IF EXISTS documents");
+    createSchema();
+  } else {
+    createSchema();
+  }
+
+  prepareStatements();
+}
+
+export function closeDb(): void {
+  db?.close();
+}
+
+export function getAllDocumentHashes(): Record<string, string> {
+  const rows = stmts.getAllDocs.all() as {
+    path: string;
+    hash: string;
+  }[];
+  const result: Record<string, string> = {};
+  for (const row of rows) {
+    result[row.path] = row.hash;
+  }
+  return result;
+}
+
+export function setDocumentHash(path: string, hash: string): void {
+  stmts.upsertDoc.run(path, hash);
+}
+
+export function removeDocument(path: string): void {
+  stmts.deleteChunksByDoc.run(path);
+  stmts.deleteDoc.run(path);
+}
+
+function embeddingToBlob(embedding: number[]): Buffer {
+  return Buffer.from(new Float32Array(embedding).buffer);
+}
+
+function blobToEmbedding(blob: Uint8Array): number[] {
+  return Array.from(new Float32Array(new Uint8Array(blob).buffer));
+}
+
+interface RawChunkRow {
+  id: number;
+  document_path: string;
+  file_heading: string;
+  heading: string;
+  text: string;
+  metadata: string;
+  embedding: Uint8Array;
+}
+
+function toChunkRow(row: RawChunkRow): ChunkRow {
+  return {
+    id: row.id,
+    path: row.document_path,
+    fileHeading: row.file_heading,
+    heading: row.heading,
+    text: row.text,
+    metadata: JSON.parse(row.metadata),
+    embedding: blobToEmbedding(row.embedding),
+  };
+}
+
+export function insertChunk(
+  documentPath: string,
+  fileHeading: string,
+  heading: string,
+  text: string,
+  metadata: Record<string, unknown>,
+  embedding: number[],
+): void {
+  stmts.insertChunk.run(
+    documentPath,
+    fileHeading,
+    heading,
+    text,
+    JSON.stringify(metadata),
+    embeddingToBlob(embedding),
+  );
+}
+
+export function getAllChunks(): ChunkRow[] {
+  const rows = stmts.getAllChunks.all() as unknown as RawChunkRow[];
+  return rows.map(toChunkRow);
+}
+
+export function getChunkCount(): number {
+  const row = stmts.countChunks.get() as {
+    count: number;
+  };
+  return row.count;
+}
+
+export function searchFts(query: string, limit: number): number[] {
+  // Sanitize: split into words, quote each to escape FTS5 operators
+  const terms = query
+    .split(/\s+/)
+    .filter((t) => t.length > 0)
+    .map((t) => `"${t.replace(/"/g, '""')}"`)
+    .join(" ");
+  if (terms.length === 0) return [];
+  try {
+    const rows = stmts.searchFts.all(terms, limit) as { rowid: number }[];
+    return rows.map((r) => r.rowid);
+  } catch {
+    // FTS5 MATCH can fail on edge-case inputs; fall back to empty
+    return [];
+  }
+}
+
+export function runInTransaction(fn: () => void): void {
+  db.exec("BEGIN");
+  try {
+    fn();
+    db.exec("COMMIT");
+  } catch (error) {
+    db.exec("ROLLBACK");
+    throw error;
+  }
+}

package/src/types.ts

@@ -0,0 +1,16 @@
+export interface BaseChunk {
+  path: string;
+  fileHeading: string;
+  heading: string;
+  text: string;
+  metadata: Record<string, unknown>;
+}
+
+export interface ChunkRow extends BaseChunk {
+  id: number;
+  embedding: number[];
+}
+
+export interface SearchResult extends Omit<ChunkRow, "id" | "embedding"> {
+  score: number;
+}

package/src/version.ts

@@ -0,0 +1,16 @@
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+export function getVersion(): string {
+  try {
+    const packageJson = JSON.parse(
+      readFileSync(join(__dirname, "..", "package.json"), "utf-8"),
+    );
+    return packageJson.version || "unknown";
+  } catch {
+    return "unknown";
+  }
+}

package/test/fixtures/docs/api.md

@@ -0,0 +1,26 @@
+---
+title: API Documentation
+version: "2.0"
+---
+
+# API Documentation
+
+## Authentication
+
+All API requests require a valid JWT token in the Authorization header.
+To obtain a token, send a POST request to `/auth/login` with your credentials.
+Tokens expire after 24 hours and must be refreshed using the `/auth/refresh` endpoint.
+
+## Rate Limiting
+
+The API enforces rate limiting to ensure fair usage across all clients.
+Default limits are 100 requests per minute for authenticated users
+and 10 requests per minute for unauthenticated access.
+Exceeding the limit returns a 429 Too Many Requests response.
+
+## Error Handling
+
+All errors follow a consistent JSON format with a `code` field and a `message` field.
+Common error codes include 400 for bad requests, 401 for unauthorized access,
+403 for forbidden resources, and 500 for internal server errors.
+Each error response includes a unique `request_id` for debugging purposes.

package/test/fixtures/docs/getting-started.md

@@ -0,0 +1,13 @@
+# Getting Started
+
+## Installation
+
+Install the package using npm. Run `npm install mylib` in your project directory.
+The package requires Node.js version 18 or higher and works on all major platforms
+including Windows, macOS, and Linux.
+
+## Configuration
+
+Create a configuration file at the root of your project named `mylib.config.json`.
+The minimum required configuration includes the `apiKey` and `endpoint` fields.
+You can also set environment variables prefixed with `MYLIB_` to override config values.

package/test/helpers/index.ts

@@ -0,0 +1,14 @@
+export function delay(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export function createMockLogger() {
+  return {
+    trace: () => {},
+    debug: () => {},
+    info: () => {},
+    warn: () => {},
+    error: () => {},
+    fatal: () => {},
+  };
+}

package/test/integration/embedder.test.ts

@@ -0,0 +1,52 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import { Embedder } from "../../src/embedder/embedder.js";
+
+describe("embedder", () => {
+  let embedder: Embedder;
+
+  it("should load the model", async () => {
+    embedder = await Embedder.load();
+    assert.ok(embedder);
+    assert.ok(embedder.maxTokens > 0);
+  });
+
+  it("should tokenize text", () => {
+    const tokens = embedder.tokenize("hello world");
+    assert.ok(Array.isArray(tokens));
+    assert.ok(tokens.length > 0);
+  });
+
+  it("should embed a single text", async () => {
+    const embedding = await embedder.embed("hello world");
+    assert.ok(Array.isArray(embedding));
+    assert.ok(embedding.length > 0);
+    assert.strictEqual(typeof embedding[0], "number");
+  });
+
+  it("should return normalized embeddings", async () => {
+    const embedding = await embedder.embed("test text");
+    const magnitude = Math.sqrt(embedding.reduce((sum, v) => sum + v * v, 0));
+    assert.ok(Math.abs(magnitude - 1.0) < 0.01);
+  });
+
+  it("should embed a batch of texts", async () => {
+    const embeddings = await embedder.embedBatch(["hello", "world"]);
+    assert.strictEqual(embeddings.length, 2);
+    assert.strictEqual(embeddings[0].length, embeddings[1].length);
+  });
+
+  it("should return empty array for empty batch", async () => {
+    const embeddings = await embedder.embedBatch([]);
+    assert.deepStrictEqual(embeddings, []);
+  });
+
+  it("should produce different embeddings for different texts", async () => {
+    const [a, b] = await embedder.embedBatch([
+      "javascript programming",
+      "cooking recipes",
+    ]);
+    const same = a.every((v, i) => v === b[i]);
+    assert.ok(!same);
+  });
+});

package/test/integration/server.test.ts

@@ -0,0 +1,125 @@
+import assert from "node:assert";
+import { resolve } from "node:path";
+import { describe, it } from "node:test";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
+
+const DOCS_PATH = resolve("test/fixtures/docs");
+
+function createTransport(): StdioClientTransport {
+  return new StdioClientTransport({
+    command: "tsx",
+    args: ["src/cli.ts"],
+    env: { ...process.env, DOCS_PATH, LOG_LEVEL: "silent" },
+  });
+}
+
+function createClient(): Client {
+  return new Client(
+    { name: "test-client", version: "1.0.0" },
+    { capabilities: {} },
+  );
+}
+
+describe("server integration", () => {
+  it("should list search_docs tool", async () => {
+    const client = createClient();
+    const transport = createTransport();
+
+    await client.connect(transport);
+    const { tools } = await client.listTools();
+
+    assert.ok(tools.length > 0);
+    assert.strictEqual(tools[0].name, "search_docs");
+
+    await client.close();
+  });
+
+  it("should search docs and return results", async () => {
+    const client = createClient();
+    const transport = createTransport();
+
+    await client.connect(transport);
+
+    const result = await client.callTool({
+      name: "search_docs",
+      arguments: { query: "how to authenticate", limit: 3 },
+    });
+
+    assert.ok(result.content);
+    assert.ok(Array.isArray(result.content));
+    assert.strictEqual(result.content[0].type, "text");
+    assert.ok(result.content[0].text.includes("Authentication"));
+
+    await client.close();
+  });
+
+  it("should clamp limit to maximum of 20", async () => {
+    const client = createClient();
+    const transport = createTransport();
+
+    await client.connect(transport);
+
+    const result = await client.callTool({
+      name: "search_docs",
+      arguments: { query: "test", limit: 999 },
+    });
+
+    assert.ok(result.content);
+    assert.ok(Array.isArray(result.content));
+
+    await client.close();
+  });
+
+  it("should clamp limit to minimum of 1", async () => {
+    const client = createClient();
+    const transport = createTransport();
+
+    await client.connect(transport);
+
+    const result = await client.callTool({
+      name: "search_docs",
+      arguments: { query: "test", limit: -5 },
+    });
+
+    assert.ok(result.content);
+    assert.ok(Array.isArray(result.content));
+
+    await client.close();
+  });
+
+  it("should use default limit when not specified", async () => {
+    const client = createClient();
+    const transport = createTransport();
+
+    await client.connect(transport);
+
+    const result = await client.callTool({
+      name: "search_docs",
+      arguments: { query: "install" },
+    });
+
+    assert.ok(result.content);
+    assert.ok(Array.isArray(result.content));
+    assert.strictEqual(result.content[0].type, "text");
+
+    await client.close();
+  });
+
+  it("should return no results message for unrelated query", async () => {
+    const client = createClient();
+    const transport = createTransport();
+
+    await client.connect(transport);
+
+    const result = await client.callTool({
+      name: "search_docs",
+      arguments: { query: "xyzzy", limit: 1 },
+    });
+
+    assert.ok(result.content);
+    assert.ok(Array.isArray(result.content));
+
+    await client.close();
+  });
+});

package/test/unit/chunker.test.ts

@@ -0,0 +1,193 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import type { ChunkOptions } from "../../src/ingest/chunker.js";
+import { chunkMarkdown } from "../../src/ingest/chunker.js";
+
+// Use character length as a stand-in for token counting in unit tests
+const defaultOptions: ChunkOptions = {
+  maxTokens: 2000,
+  countTokens: (text: string) => text.length,
+};
+
+describe("chunker", () => {
+  it("should split on H2 headings", () => {
+    const md = `# My Doc
+
+## First Section
+
+This is the first section with enough content to pass the minimum length filter for chunks.
+
+## Second Section
+
+This is the second section with enough content to pass the minimum length filter for chunks.
+`;
+    const chunks = chunkMarkdown(md, "test.md", defaultOptions);
+    assert.strictEqual(chunks.length, 2);
+    assert.strictEqual(chunks[0].heading, "First Section");
+    assert.strictEqual(chunks[1].heading, "Second Section");
+  });
+
+  it("should extract H1 as fileHeading", () => {
+    const md = `# API Reference
+
+## Methods
+
+Here are the available methods with detailed descriptions and usage examples for each one.
+`;
+    const chunks = chunkMarkdown(md, "api.md", defaultOptions);
+    assert.strictEqual(chunks[0].fileHeading, "API Reference");
+  });
+
+  it("should fall back to filename when no H1", () => {
+    const md = `## Only Section
+
+Content here with enough text to pass the minimum length requirement for the chunk filter.
+`;
+    const chunks = chunkMarkdown(md, "guide.md", defaultOptions);
+    assert.strictEqual(chunks[0].fileHeading, "guide");
+  });
+
+  it("should parse frontmatter", () => {
+    const md = `---
+title: My Guide
+version: "1.0"
+---
+
+# Guide
+
+## Section One
+
+This section has enough content to be included as a chunk in the final output results.
+`;
+    const chunks = chunkMarkdown(md, "guide.md", defaultOptions);
+    assert.strictEqual(chunks[0].metadata.title, "My Guide");
+    assert.strictEqual(chunks[0].metadata.version, "1.0");
+  });
+
+  it("should skip empty sections", () => {
+    const md = `# Doc
+
+## Empty
+
+## Has Content
+
+This section has actual content that should be included in the output.
+`;
+    const chunks = chunkMarkdown(md, "test.md", defaultOptions);
+    assert.strictEqual(chunks.length, 1);
+    assert.strictEqual(chunks[0].heading, "Has Content");
+  });
+
+  it("should strip HTML comments", () => {
+    const md = `# Doc
+
+## Section
+
+<!-- This is a comment that should be removed -->
+The actual content remains here and has enough length to pass the minimum filter requirement.
+`;
+    const chunks = chunkMarkdown(md, "test.md", defaultOptions);
+    assert.ok(!chunks[0].text.includes("<!--"));
+    assert.ok(chunks[0].text.includes("actual content"));
+  });
+
+  it("should sub-split oversized sections", () => {
+    const paragraph =
+      "This is a paragraph with enough words to take up space. ".repeat(30);
+    const md = `# Doc
+
+## Big Section
+
+${paragraph}
+
+### Sub Heading A
+
+${paragraph}
+
+### Sub Heading B
+
+${paragraph}
+`;
+    const chunks = chunkMarkdown(md, "test.md", defaultOptions);
+    assert.ok(
+      chunks.length > 1,
+      `Expected multiple chunks, got ${chunks.length}`,
+    );
+    for (const chunk of chunks) {
+      assert.ok(
+        chunk.text.length <= 2200,
+        `Chunk too large: ${chunk.text.length} chars`,
+      );
+      assert.strictEqual(chunk.heading, "Big Section");
+    }
+  });
+
+  it("should keep small sections as single chunks", () => {
+    const md = `# Doc
+
+## Small Section
+
+This section is well under the limit and should remain as a single chunk without splitting.
+`;
+    const chunks = chunkMarkdown(md, "test.md", defaultOptions);
+    assert.strictEqual(chunks.length, 1);
+  });
+
+  it("should preserve heading for all sub-chunks", () => {
+    const longText = "Sentence one with content. ".repeat(200);
+    const md = `# Doc
+
+## API Reference
+
+${longText}
+`;
+    const chunks = chunkMarkdown(md, "test.md", defaultOptions);
+    assert.ok(chunks.length > 1);
+    for (const chunk of chunks) {
+      assert.strictEqual(chunk.heading, "API Reference");
+      assert.strictEqual(chunk.fileHeading, "Doc");
+    }
+  });
+
+  it("should respect custom measure function for splitting", () => {
+    const md = `# Doc
+
+## Section
+
+${"word ".repeat(100)}
+`;
+    // Count by words, max 20 words per chunk
+    const wordOptions: ChunkOptions = {
+      maxTokens: 20,
+      countTokens: (text: string) => text.split(/\s+/).filter(Boolean).length,
+    };
+    const chunks = chunkMarkdown(md, "test.md", wordOptions);
+    assert.ok(
+      chunks.length > 1,
+      `Expected multiple chunks, got ${chunks.length}`,
+    );
+    for (const chunk of chunks) {
+      const wordCount = chunk.text.split(/\s+/).filter(Boolean).length;
+      assert.ok(
+        wordCount <= 25,
+        `Chunk has ${wordCount} words, expected <= 25 (with overlap)`,
+      );
+    }
+  });
+
+  it("should include pre-H2 content with fileHeading", () => {
+    const md = `# My Doc
+
+This is intro content before any H2 heading.
+
+## First Section
+
+Section content here.
+`;
+    const chunks = chunkMarkdown(md, "test.md", defaultOptions);
+    assert.strictEqual(chunks.length, 2);
+    assert.strictEqual(chunks[0].heading, "My Doc");
+    assert.ok(chunks[0].text.includes("intro content"));
+    assert.strictEqual(chunks[1].heading, "First Section");
+  });
+});