@tideshift/sextant 0.1.0

package/.env.example

@@ -0,0 +1,25 @@
+# Path to the docs folder to index (absolute or relative to project root)
+DOCS_PATH=./docs
+
+# Ollama endpoint
+OLLAMA_URL=http://localhost:11434
+
+# Embedding model
+EMBEDDING_MODEL=nomic-embed-text
+
+# Embedding dimensions (must match model; nomic-embed-text = 768, mxbai-embed-large = 1024)
+EMBEDDING_DIMS=768
+
+# Data storage path (for Orama index + SQLite metadata)
+DATA_PATH=.sextant
+
+# Chunk size limits
+MAX_CHUNK_TOKENS=512
+CHUNK_OVERLAP_LINES=2
+
+# Search defaults
+DEFAULT_TOP_K=10
+
+# Hybrid search weights (0.0-1.0, must sum to 1.0)
+HYBRID_WEIGHT_TEXT=0.5
+HYBRID_WEIGHT_VECTOR=0.5

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Tideshift Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,123 @@
+<p align="center">
+  <img src="docs/sextant-logo.png" alt="Sextant" width="300">
+</p>
+
+<p align="center">
+  <a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="License: MIT"></a>
+  <img src="https://img.shields.io/badge/runtime-Bun-f9f1e1?logo=bun" alt="Bun">
+  <img src="https://img.shields.io/badge/protocol-MCP-7c3aed" alt="MCP">
+</p>
+
+# Sextant
+
+Sextant is an MCP server that provides hybrid semantic and keyword search over a local folder of markdown files. It indexes your docs, chunks them by heading, embeds them through a local Ollama model, and exposes search and retrieval tools over the Model Context Protocol (MCP).
+
+Claude Code (or any MCP client) can then search your documentation using natural language queries, exact keyword lookups, or a combination of both.
+
+## How it works
+
+1. Sextant scans a folder of markdown files and splits them into chunks at each heading boundary.
+2. Each chunk is embedded locally using Ollama (nomic-embed-text).
+3. Chunks and their embeddings are stored in Orama, which handles full-text, vector, and hybrid search in a single library.
+4. File metadata is tracked in SQLite (via bun:sqlite) so unchanged files are skipped on restart.
+5. Each search request checks for stale files and triggers a background reindex if needed.
+6. The MCP server exposes five tools over stdio: `search_docs`, `list_docs`, `get_doc`, `reindex_docs`, and `sextant_status`.
+
+The entire stack runs locally with no external services. There are no native modules; everything is pure JavaScript/TypeScript.
+
+### Index freshness
+
+Sextant keeps its index up to date automatically:
+
+- On startup, Sextant compares every file's modification time against what was previously indexed. New and changed files are re-indexed; deleted files are removed from the index. This covers cases where docs were updated while the server was not running (for example, after a `git pull`).
+- On each search, a freshness check detects new, modified, or deleted files and triggers a background reindex without blocking the query. If files have changed, the response includes a note and results update on the next search.
+- If another Sextant process has persisted a newer index to disk, it is picked up automatically before searching.
+
+## Prerequisites
+
+### Bun
+
+Sextant runs on [Bun](https://bun.sh), which is used as the runtime, package manager, and bundler.
+
+**Windows (PowerShell):**
+
+```powershell
+powershell -c "irm bun.sh/install.ps1 | iex"
+```
+
+**macOS / Linux:**
+
+```bash
+curl -fsSL https://bun.sh/install | bash
+```
+
+Verify the installation with `bun --version`.
+
+### Ollama
+
+[Ollama](https://ollama.com) must be installed and running locally. Pull the embedding model before first use:
+
+```bash
+ollama pull nomic-embed-text
+```
+
+Ollama runs on `http://localhost:11434` by default. If Ollama is not running, keyword search will still work, but semantic and hybrid search will be unavailable.
+
+## Usage with Claude Code
+
+Add the following to your Claude Code MCP configuration (`.claude/mcp.json` in your project):
+
+```json
+{
+  "mcpServers": {
+    "sextant": {
+      "type": "stdio",
+      "command": "bunx",
+      "args": ["@tideshift/sextant"],
+      "env": {
+        "DOCS_PATH": "/absolute/path/to/your/docs"
+      }
+    }
+  }
+}
+```
+
+Set `DOCS_PATH` to the absolute path of the markdown folder you want to index.
+
+Once configured, Claude Code will have access to the following tools:
+
+- **search_docs** - Search documentation using hybrid semantic + keyword search. Supports three modes: `hybrid` (default, combines both), `semantic` (vector similarity only), and `keyword` (exact text matching only). Accepts an optional category filter.
+- **list_docs** - List all indexed documents with their categories, titles, and chunk counts.
+- **get_doc** - Retrieve the full content of a specific document by file path.
+- **reindex_docs** - Force a full re-index of all documents.
+- **sextant_status** - Check server health, indexing progress, Ollama connectivity, and index statistics.
+
+## Configuration
+
+All settings can be configured through environment variables or a `.env` file:
+
+| Variable | Default | Description |
+|---|---|---|
+| `DOCS_PATH` | `./docs` | Path to the markdown folder to index |
+| `OLLAMA_URL` | `http://localhost:11434` | Ollama API endpoint |
+| `EMBEDDING_MODEL` | `nomic-embed-text` | Ollama embedding model name |
+| `EMBEDDING_DIMS` | `768` | Embedding vector dimensions (must match model) |
+| `DATA_PATH` | `.sextant` | Where to store the index and metadata |
+| `MAX_CHUNK_TOKENS` | `512` | Maximum chunk size (estimated as chars / 4) |
+| `DEFAULT_TOP_K` | `10` | Default number of search results |
+| `HYBRID_WEIGHT_TEXT` | `0.5` | Weight for keyword matching in hybrid mode |
+| `HYBRID_WEIGHT_VECTOR` | `0.5` | Weight for semantic matching in hybrid mode |
+
+## Contributing
+
+See [docs/contributing.md](docs/contributing.md) for local development setup, building, and diagnostics.
+
+## Technical details
+
+See [docs/technical_details.md](docs/technical_details.md) for architecture, project structure, chunking strategy, persistence model, index freshness logic, and edge cases.
+
+## License
+
+Made with ❤️ in Vancouver, BC by Tideshift Labs, developed using [Claude Code](https://claude.ai/code).
+
+[MIT](LICENSE)

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@tideshift/sextant",
+  "version": "0.1.0",
+  "description": "MCP server for hybrid semantic + keyword search over local markdown docs",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Tideshift-Labs/sextant.git"
+  },
+  "homepage": "https://github.com/Tideshift-Labs/sextant",
+  "keywords": [
+    "mcp",
+    "search",
+    "documentation",
+    "semantic-search",
+    "ollama",
+    "embeddings",
+    "orama",
+    "markdown",
+    "claude",
+    "model-context-protocol"
+  ],
+  "module": "src/index.ts",
+  "type": "module",
+  "bin": {
+    "sextant": "src/index.ts"
+  },
+  "files": [
+    "src/**/*",
+    ".env.example"
+  ],
+  "scripts": {
+    "dev": "bun run src/index.ts",
+    "start": "bun run src/index.ts",
+    "build": "bun build --compile --minify src/index.ts --outfile dist/docs-mcp-server",
+    "typecheck": "bunx tsc --noEmit",
+    "test": "bun test",
+    "reindex": "bun run src/scripts/reindex.ts"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "@types/node": "^25.5.0",
+    "bun-types": "^1.3.10"
+  },
+  "peerDependencies": {
+    "typescript": "^5"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.27.1",
+    "@orama/orama": "^3.1.18",
+    "dotenv": "^17.3.1"
+  }
+}

package/src/config.ts

@@ -0,0 +1,50 @@
+import dotenv from 'dotenv';
+import path from 'path';
+
+dotenv.config();
+
+function envStr(key: string, fallback: string): string {
+  return process.env[key] ?? fallback;
+}
+
+function envInt(key: string, fallback: number): number {
+  const v = process.env[key];
+  return v ? parseInt(v, 10) : fallback;
+}
+
+function envFloat(key: string, fallback: number): number {
+  const v = process.env[key];
+  return v ? parseFloat(v) : fallback;
+}
+
+function envBool(key: string, fallback: boolean): boolean {
+  const v = process.env[key];
+  if (!v) return fallback;
+  return v.toLowerCase() === 'true' || v === '1';
+}
+
+export const config = {
+  docsPath: path.resolve(envStr('DOCS_PATH', './docs')),
+  ollamaUrl: envStr('OLLAMA_URL', 'http://localhost:11434'),
+  embeddingModel: envStr('EMBEDDING_MODEL', 'nomic-embed-text'),
+  embeddingDims: envInt('EMBEDDING_DIMS', 768),
+  dataPath: path.resolve(envStr('DATA_PATH', '.sextant')),
+  maxChunkTokens: envInt('MAX_CHUNK_TOKENS', 512),
+  chunkOverlapLines: envInt('CHUNK_OVERLAP_LINES', 2),
+  defaultTopK: envInt('DEFAULT_TOP_K', 10),
+  hybridWeightText: envFloat('HYBRID_WEIGHT_TEXT', 0.5),
+  hybridWeightVector: envFloat('HYBRID_WEIGHT_VECTOR', 0.5),
+  similarityThreshold: envFloat('SIMILARITY_THRESHOLD', 0.5),
+
+  // Embedding instructions (nomic-embed-text uses "search_document: " / "search_query: " prefixes)
+  indexInstruction: envStr('INDEX_INSTRUCTION', 'search_document: '),
+  queryInstruction: envStr('QUERY_INSTRUCTION', 'search_query: '),
+
+  // Persistence debounce
+  persistDebounceMs: envInt('PERSIST_DEBOUNCE_MS', 5000),
+
+  // Embedding batch size
+  embeddingBatchSize: envInt('EMBEDDING_BATCH_SIZE', 32),
+} as const;
+
+export type Config = typeof config;

package/src/ignore-files.ts

@@ -0,0 +1,31 @@
+import { existsSync, readFileSync, appendFileSync } from 'fs';
+import path from 'path';
+
+const SEXTANT_ENTRY = '.sextant/';
+const IGNORE_FILES = ['.gitignore', '.dvignore', '.p4ignore', '.hgignore'];
+
+/**
+ * For each known ignore file that exists in the project root,
+ * append `.sextant/` if it isn't already listed.
+ */
+export function ensureIgnored(projectRoot: string): void {
+  for (const file of IGNORE_FILES) {
+    const filePath = path.join(projectRoot, file);
+    if (!existsSync(filePath)) continue;
+
+    try {
+      const content = readFileSync(filePath, 'utf-8');
+      // Check if already ignored (exact line match)
+      const lines = content.split(/\r?\n/);
+      if (lines.some((line) => line.trim() === SEXTANT_ENTRY || line.trim() === '.sextant')) {
+        continue;
+      }
+
+      const suffix = content.endsWith('\n') ? '' : '\n';
+      appendFileSync(filePath, `${suffix}${SEXTANT_ENTRY}\n`);
+      console.error(`[ignore] Added ${SEXTANT_ENTRY} to ${file}`);
+    } catch (err) {
+      console.error(`[ignore] Could not update ${file}:`, err);
+    }
+  }
+}

package/src/index.ts

@@ -0,0 +1,69 @@
+#!/usr/bin/env bun
+import { mkdirSync } from 'fs';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { config } from './config.ts';
+import { initMetadataDb, closeMetadataDb } from './store/metadata-db.ts';
+import { initStore } from './store/orama-store.ts';
+import { loadFromDisk, flushPersist } from './store/persistence.ts';
+import { indexAll } from './indexer/pipeline.ts';
+import { requestCancel } from './indexer/state.ts';
+import { createMcpServer } from './server.ts';
+import { ensureIgnored } from './ignore-files.ts';
+
+async function main() {
+  console.error('[startup] sextant starting...');
+  console.error(`[startup] Docs path: ${config.docsPath}`);
+  console.error(`[startup] Data path: ${config.dataPath}`);
+
+  // Ensure directories exist
+  mkdirSync(config.dataPath, { recursive: true });
+  mkdirSync(config.docsPath, { recursive: true });
+
+  // Add .sextant/ to ignore files if they exist in cwd
+  ensureIgnored(process.cwd());
+
+  // Initialize metadata DB
+  initMetadataDb();
+  console.error('[startup] Metadata DB initialized');
+
+  // Load persisted index from disk, or create fresh
+  const loaded = await loadFromDisk();
+  if (!loaded) {
+    await initStore();
+    console.error('[startup] Created fresh Orama index');
+  }
+
+  // Start MCP server immediately so the connection is available while indexing
+  const server = createMcpServer();
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+  console.error('[startup] MCP server running on stdio');
+
+  // Run incremental reindex in background (only re-embeds changed files)
+  const indexingPromise = indexAll(config.docsPath)
+    .then((stats) => {
+      console.error(`[startup] Indexing complete: ${stats.filesProcessed} files, ${stats.chunksCreated} chunks in ${stats.duration}ms`);
+    })
+    .catch((err) => {
+      console.error('[startup] Initial indexing failed (server keeps running):', err);
+    });
+
+  // Graceful shutdown
+  const shutdown = async () => {
+    console.error('[shutdown] Shutting down...');
+    requestCancel();
+    // Give indexing up to 2s to finish gracefully
+    await Promise.race([indexingPromise, Bun.sleep(2000)]);
+    await flushPersist();
+    closeMetadataDb();
+    process.exit(0);
+  };
+
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+}
+
+main().catch((err) => {
+  console.error('Fatal error:', err);
+  process.exit(1);
+});

package/src/indexer/chunker.ts

@@ -0,0 +1,245 @@
+import path from 'path';
+import { createHash } from 'crypto';
+import { config } from '../config.ts';
+import type { DocChunk, ChunkMetadata } from './types.ts';
+
+const MAX_CHUNK_CHARS = config.maxChunkTokens * 4; // ~1 token ≈ 4 chars
+
+interface FrontmatterResult {
+  metadata: ChunkMetadata;
+  bodyStartIndex: number;
+}
+
+function parseFrontmatter(lines: string[]): FrontmatterResult {
+  const metadata: ChunkMetadata = {};
+  if (lines[0]?.trim() !== '---') {
+    return { metadata, bodyStartIndex: 0 };
+  }
+
+  for (let i = 1; i < lines.length; i++) {
+    if (lines[i]!.trim() === '---') {
+      // Parse YAML-like frontmatter between the --- delimiters
+      for (let j = 1; j < i; j++) {
+        const line = lines[j]!;
+        const match = line.match(/^(\w+)\s*:\s*(.+)$/);
+        if (match) {
+          const [, key, value] = match;
+          if (key === 'title') metadata.title = value!.replace(/^["']|["']$/g, '');
+          if (key === 'category') metadata.category = value!.replace(/^["']|["']$/g, '');
+          if (key === 'tags') {
+            metadata.tags = value!
+              .replace(/^\[|\]$/g, '')
+              .split(',')
+              .map((t) => t.trim().replace(/^["']|["']$/g, ''));
+          }
+        }
+      }
+      return { metadata, bodyStartIndex: i + 1 };
+    }
+  }
+  return { metadata, bodyStartIndex: 0 };
+}
+
+function makeChunkId(filePath: string, content: string): string {
+  const input = `${filePath}::${content}`;
+  return createHash('sha256').update(input).digest('hex').slice(0, 16);
+}
+
+function getHeadingLevel(line: string): number {
+  const match = line.match(/^(#{1,6})\s/);
+  return match ? match[1]!.length : 0;
+}
+
+function getHeadingText(line: string): string {
+  return line.replace(/^#{1,6}\s+/, '').trim();
+}
+
+function isInsideCodeBlock(lines: string[], index: number): boolean {
+  let fenceCount = 0;
+  for (let i = 0; i < index; i++) {
+    if (lines[i]!.trimStart().startsWith('```')) {
+      fenceCount++;
+    }
+  }
+  return fenceCount % 2 === 1;
+}
+
+interface HeadingSection {
+  headingHierarchy: string[];
+  lines: string[];
+}
+
+function splitByHeadings(lines: string[]): HeadingSection[] {
+  const sections: HeadingSection[] = [];
+  const hierarchyStack: { level: number; text: string }[] = [];
+  let currentLines: string[] = [];
+  let hasHeading = false;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i]!;
+    const level = getHeadingLevel(line);
+
+    if (level > 0 && !isInsideCodeBlock(lines, i)) {
+      // Save previous section
+      if (currentLines.length > 0 || hasHeading) {
+        sections.push({
+          headingHierarchy: hierarchyStack.map((h) => h.text),
+          lines: currentLines,
+        });
+      }
+
+      // Update heading hierarchy
+      while (hierarchyStack.length > 0 && hierarchyStack[hierarchyStack.length - 1]!.level >= level) {
+        hierarchyStack.pop();
+      }
+      hierarchyStack.push({ level, text: getHeadingText(line) });
+
+      currentLines = [line];
+      hasHeading = true;
+    } else {
+      currentLines.push(line);
+    }
+  }
+
+  // Push final section
+  if (currentLines.length > 0) {
+    sections.push({
+      headingHierarchy: hasHeading ? hierarchyStack.map((h) => h.text) : [],
+      lines: currentLines,
+    });
+  }
+
+  return sections;
+}
+
+function splitOversizedParagraph(para: string): string[] {
+  // Hard-split a single paragraph that exceeds MAX_CHUNK_CHARS by lines
+  const lines = para.split('\n');
+  const pieces: string[] = [];
+  let current = '';
+
+  for (const line of lines) {
+    // If a single line exceeds the limit, truncate it
+    const safeLine = line.length > MAX_CHUNK_CHARS ? line.slice(0, MAX_CHUNK_CHARS) : line;
+
+    if (current.length + safeLine.length + 1 > MAX_CHUNK_CHARS && current.length > 0) {
+      pieces.push(current.trim());
+      current = safeLine;
+    } else {
+      current = current ? current + '\n' + safeLine : safeLine;
+    }
+  }
+
+  if (current.trim()) {
+    pieces.push(current.trim());
+  }
+
+  return pieces.length > 0 ? pieces : [para];
+}
+
+function splitLargeChunk(content: string, overlapLines: number): string[] {
+  if (content.length <= MAX_CHUNK_CHARS) return [content];
+
+  const paragraphs = content.split(/\n\n+/);
+  const subChunks: string[] = [];
+  let current = '';
+
+  for (const para of paragraphs) {
+    // If a single paragraph exceeds the limit, hard-split it by lines
+    if (para.length > MAX_CHUNK_CHARS) {
+      if (current.trim()) {
+        subChunks.push(current.trim());
+        current = '';
+      }
+      subChunks.push(...splitOversizedParagraph(para));
+      continue;
+    }
+
+    if (current.length + para.length + 2 > MAX_CHUNK_CHARS && current.length > 0) {
+      subChunks.push(current.trim());
+      // Add overlap: last N lines of previous chunk
+      const prevLines = current.trimEnd().split('\n');
+      const overlap = prevLines.slice(-overlapLines).join('\n');
+      current = overlap ? overlap + '\n\n' + para : para;
+    } else {
+      current = current ? current + '\n\n' + para : para;
+    }
+  }
+
+  if (current.trim()) {
+    subChunks.push(current.trim());
+  }
+
+  return subChunks.length > 0 ? subChunks : [content];
+}
+
+export function chunkMarkdown(
+  filePath: string,
+  rawContent: string,
+  lastModified: number,
+  docsPath: string
+): { chunks: DocChunk[]; metadata: ChunkMetadata } {
+  const relativePath = path.relative(docsPath, filePath).replace(/\\/g, '/');
+  const fileName = path.basename(filePath);
+  const parts = relativePath.split('/');
+  const category = parts.length > 1 ? parts[0]! : 'root';
+
+  const lines = rawContent.split('\n');
+  const { metadata, bodyStartIndex } = parseFrontmatter(lines);
+  const bodyLines = lines.slice(bodyStartIndex);
+  const fileCategory = metadata.category ?? category;
+
+  const sections = splitByHeadings(bodyLines);
+  const chunks: DocChunk[] = [];
+
+  // If no headings found, treat entire file as one chunk
+  if (sections.length === 0 || (sections.length === 1 && sections[0]!.headingHierarchy.length === 0)) {
+    const content = bodyLines.join('\n').trim();
+    if (content) {
+      const hierarchy = [metadata.title ?? fileName.replace(/\.md$/, '')];
+      const subChunks = splitLargeChunk(content, config.chunkOverlapLines);
+      for (let i = 0; i < subChunks.length; i++) {
+        chunks.push({
+          id: makeChunkId(relativePath, subChunks[i]!),
+          filePath: relativePath,
+          fileName,
+          category: fileCategory,
+          headingHierarchy: hierarchy,
+          headingSlug: hierarchy.join(' > '),
+          chunkIndex: i,
+          content: subChunks[i]!,
+          charCount: subChunks[i]!.length,
+          lastModified,
+        });
+      }
+    }
+    return { chunks, metadata };
+  }
+
+  for (const section of sections) {
+    const content = section.lines.join('\n').trim();
+    if (!content) continue;
+
+    const hierarchy = section.headingHierarchy.length > 0
+      ? section.headingHierarchy
+      : [metadata.title ?? fileName.replace(/\.md$/, '')];
+
+    const subChunks = splitLargeChunk(content, config.chunkOverlapLines);
+    for (let i = 0; i < subChunks.length; i++) {
+      chunks.push({
+        id: makeChunkId(relativePath, subChunks[i]!),
+        filePath: relativePath,
+        fileName,
+        category: fileCategory,
+        headingHierarchy: hierarchy,
+        headingSlug: hierarchy.join(' > '),
+        chunkIndex: i,
+        content: subChunks[i]!,
+        charCount: subChunks[i]!.length,
+        lastModified,
+      });
+    }
+  }
+
+  return { chunks, metadata };
+}