@tideshift/sextant 0.1.0

package/src/indexer/embedder.ts

@@ -0,0 +1,79 @@
+import { config } from '../config.ts';
+
+const MAX_RETRIES = 3;
+const INITIAL_BACKOFF_MS = 500;
+
+interface OllamaEmbedResponse {
+  embeddings: number[][];
+}
+
+async function callOllamaEmbed(inputs: string[]): Promise<number[][]> {
+  let lastError: Error | null = null;
+
+  for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+    try {
+      const response = await fetch(`${config.ollamaUrl}/api/embed`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          model: config.embeddingModel,
+          input: inputs,
+        }),
+      });
+
+      if (!response.ok) {
+        throw new Error(`Ollama returned ${response.status}: ${await response.text()}`);
+      }
+
+      const data = (await response.json()) as OllamaEmbedResponse;
+      return data.embeddings;
+    } catch (err) {
+      lastError = err instanceof Error ? err : new Error(String(err));
+      if (attempt < MAX_RETRIES - 1) {
+        const delay = INITIAL_BACKOFF_MS * Math.pow(2, attempt);
+        await Bun.sleep(delay);
+      }
+    }
+  }
+
+  throw lastError;
+}
+
+export async function checkOllamaHealth(): Promise<boolean> {
+  try {
+    const response = await fetch(`${config.ollamaUrl}/api/tags`);
+    return response.ok;
+  } catch {
+    return false;
+  }
+}
+
+export async function embedTexts(texts: string[], isQuery = false): Promise<number[][]> {
+  const instruction = isQuery ? config.queryInstruction : config.indexInstruction;
+  const prefixedTexts = texts.map((t) => instruction + t);
+
+  const allEmbeddings: number[][] = [];
+
+  for (let i = 0; i < prefixedTexts.length; i += config.embeddingBatchSize) {
+    const batch = prefixedTexts.slice(i, i + config.embeddingBatchSize);
+    try {
+      const embeddings = await callOllamaEmbed(batch);
+      allEmbeddings.push(...embeddings);
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      if (msg.includes('fetch') || msg.includes('ECONNREFUSED') || msg.includes('Failed')) {
+        throw new Error(
+          `Ollama is not reachable at ${config.ollamaUrl}. Ensure Ollama is running with '${config.embeddingModel}' model pulled (run: ollama pull ${config.embeddingModel}).`
+        );
+      }
+      throw err;
+    }
+  }
+
+  return allEmbeddings;
+}
+
+export async function embedQuery(query: string): Promise<number[]> {
+  const results = await embedTexts([query], true);
+  return results[0]!;
+}

package/src/indexer/freshness.ts

@@ -0,0 +1,74 @@
+import path from 'path';
+import { Glob } from 'bun';
+import { config } from '../config.ts';
+import { getFile, getAllFiles } from '../store/metadata-db.ts';
+import { getState } from './state.ts';
+import { indexAll } from './pipeline.ts';
+import { reloadIfChanged } from '../store/persistence.ts';
+
+export interface FreshnessResult {
+  stale: boolean;
+  staleCount?: number;
+  alreadyIndexing?: boolean;
+}
+
+export async function checkAndReindex(): Promise<FreshnessResult> {
+  // Pick up any index persisted by another process first
+  await reloadIfChanged();
+
+  const docsPath = config.docsPath;
+
+  // Scan disk for current markdown files
+  const glob = new Glob('**/*.md');
+  const diskFiles = new Set<string>();
+  for await (const match of glob.scan({ cwd: docsPath, absolute: false })) {
+    diskFiles.add(match);
+  }
+
+  // Check for deleted files
+  const indexedFiles = getAllFiles();
+  const indexedSet = new Set(indexedFiles.map((f) => f.filePath));
+  let staleCount = 0;
+
+  for (const indexed of indexedFiles) {
+    if (!diskFiles.has(indexed.filePath)) {
+      staleCount++;
+    }
+  }
+
+  // Check for new or modified files
+  for (const relPath of diskFiles) {
+    const absPath = path.join(docsPath, relPath);
+    try {
+      const stat = await Bun.file(absPath).stat();
+      if (!stat) continue;
+
+      const existing = getFile(relPath);
+      if (!existing) {
+        // New file
+        staleCount++;
+      } else if (stat.mtimeMs > existing.lastModified) {
+        // Modified file
+        staleCount++;
+      }
+    } catch {
+      // File disappeared between scan and stat, count as stale
+      if (indexedSet.has(relPath)) staleCount++;
+    }
+  }
+
+  if (staleCount === 0) {
+    return { stale: false };
+  }
+
+  if (getState().status === 'indexing') {
+    return { stale: true, staleCount, alreadyIndexing: true };
+  }
+
+  // Fire-and-forget background reindex
+  indexAll(docsPath).catch((err) => {
+    console.error('[freshness] Background reindex failed:', err);
+  });
+
+  return { stale: true, staleCount };
+}

package/src/indexer/pipeline.ts

@@ -0,0 +1,161 @@
+import path from 'path';
+import { Glob } from 'bun';
+import { config } from '../config.ts';
+import { chunkMarkdown } from './chunker.ts';
+import { embedTexts, checkOllamaHealth } from './embedder.ts';
+import { insertChunks, removeByIds, removeByFile, initStore } from '../store/orama-store.ts';
+import { upsertFile, removeFile as removeFileMeta, getFile, getChunkIds, getAllFiles, clearAll as clearMetadata } from '../store/metadata-db.ts';
+import { persistToDisk } from '../store/persistence.ts';
+import { setIndexing, updateProgress, setReady, setError, isCancelRequested, getState } from './state.ts';
+import type { IndexStats, DocChunk } from './types.ts';
+
+export async function indexAll(docsPath: string): Promise<IndexStats> {
+  const start = Date.now();
+  let filesProcessed = 0;
+  let chunksCreated = 0;
+
+  // Collect all markdown files
+  const glob = new Glob('**/*.md');
+  const filePaths: string[] = [];
+  for await (const match of glob.scan({ cwd: docsPath, absolute: false })) {
+    filePaths.push(match);
+  }
+
+  // Remove files from the index that no longer exist on disk
+  const diskFileSet = new Set(filePaths);
+  const indexedFiles = getAllFiles();
+  let deletionsPerformed = false;
+  for (const indexed of indexedFiles) {
+    if (!diskFileSet.has(indexed.filePath)) {
+      console.error(`[pipeline] Removing deleted file from index: ${indexed.filePath}`);
+      const knownIds = getChunkIds(indexed.filePath);
+      if (knownIds) {
+        await removeByIds(knownIds);
+      } else {
+        await removeByFile(indexed.filePath);
+      }
+      removeFileMeta(indexed.filePath);
+      deletionsPerformed = true;
+    }
+  }
+
+  setIndexing(filePaths.length);
+
+  if (filePaths.length === 0) {
+    console.error('[pipeline] No markdown files found in', docsPath);
+    setReady({ filesProcessed: 0, chunksCreated: 0 });
+    return { filesProcessed: 0, chunksCreated: 0, duration: Date.now() - start };
+  }
+
+  console.error(`[pipeline] Found ${filePaths.length} markdown files`);
+
+  // Check Ollama availability
+  const ollamaOk = await checkOllamaHealth();
+  if (!ollamaOk) {
+    console.error(`[pipeline] WARNING: Ollama not reachable at ${config.ollamaUrl}. Indexing will fail for embeddings.`);
+    console.error(`[pipeline] Run: ollama pull ${config.embeddingModel}`);
+  }
+
+  // Chunk all files
+  const allChunks: DocChunk[] = [];
+  const fileMetadata: { filePath: string; lastModified: number; chunkCount: number; chunkIds: string[]; title: string | null; category: string }[] = [];
+
+  for (const relPath of filePaths) {
+    if (isCancelRequested()) {
+      console.error('[pipeline] Indexing cancelled');
+      setError('Indexing cancelled');
+      return { filesProcessed, chunksCreated, duration: Date.now() - start };
+    }
+
+    updateProgress(filesProcessed, chunksCreated, relPath);
+    const absPath = path.join(docsPath, relPath);
+    try {
+      const file = Bun.file(absPath);
+      const stat = await file.stat();
+      if (!stat) continue;
+      const lastModified = stat.mtimeMs;
+
+      // Check if file needs re-indexing
+      const existing = getFile(relPath);
+      if (existing && existing.lastModified >= lastModified) {
+        console.error(`[pipeline] Skipping unchanged: ${relPath}`);
+        continue;
+      }
+
+      const content = await file.text();
+      const { chunks, metadata } = chunkMarkdown(absPath, content, lastModified, docsPath);
+
+      if (chunks.length > 0) {
+        // Remove old chunks before inserting to avoid duplicate ID collisions
+        const knownIds = getChunkIds(relPath);
+        if (knownIds) {
+          await removeByIds(knownIds);
+        } else {
+          await removeByFile(relPath);
+        }
+        allChunks.push(...chunks);
+        fileMetadata.push({
+          filePath: relPath,
+          lastModified,
+          chunkCount: chunks.length,
+          chunkIds: chunks.map((c) => c.id),
+          title: metadata.title ?? chunks[0]?.headingHierarchy[0] ?? null,
+          category: chunks[0]?.category ?? 'root',
+        });
+      }
+
+      filesProcessed++;
+    } catch (err) {
+      console.error(`[pipeline] Error processing ${relPath}:`, err);
+    }
+  }
+
+  if (allChunks.length === 0) {
+    console.error('[pipeline] No new chunks to index');
+    if (deletionsPerformed) await persistToDisk();
+    setReady({ filesProcessed, chunksCreated: 0 });
+    return { filesProcessed, chunksCreated: 0, duration: Date.now() - start };
+  }
+
+  // Embed all chunks in batches
+  console.error(`[pipeline] Embedding ${allChunks.length} chunks...`);
+  try {
+    const texts = allChunks.map((c) => c.content);
+    const embeddings = await embedTexts(texts);
+
+    // Insert into Orama
+    await insertChunks(allChunks, embeddings);
+    chunksCreated = allChunks.length;
+
+    // Update metadata DB
+    for (const fm of fileMetadata) {
+      upsertFile(fm.filePath, fm.lastModified, fm.chunkCount, fm.title, fm.category, fm.chunkIds);
+    }
+
+    // Persist
+    await persistToDisk();
+
+    console.error(`[pipeline] Indexed ${filesProcessed} files, ${chunksCreated} chunks in ${Date.now() - start}ms`);
+    setReady({ filesProcessed, chunksCreated });
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    setError(msg);
+    console.error('[pipeline] Embedding/indexing failed:', err);
+    throw err;
+  }
+
+  return { filesProcessed, chunksCreated, duration: Date.now() - start };
+}
+
+export async function fullReindex(docsPath: string, clearExisting: boolean): Promise<IndexStats> {
+  if (getState().status === 'indexing') {
+    return { filesProcessed: 0, chunksCreated: 0, duration: 0 };
+  }
+
+  if (clearExisting) {
+    console.error('[pipeline] Clearing existing index...');
+    await initStore();
+    clearMetadata();
+  }
+  return indexAll(docsPath);
+}

package/src/indexer/state.ts

@@ -0,0 +1,70 @@
+export type IndexingStatus = 'idle' | 'indexing' | 'ready' | 'error';
+
+export interface IndexingState {
+  status: IndexingStatus;
+  filesFound: number;
+  filesProcessed: number;
+  chunksCreated: number;
+  currentFile: string | null;
+  startedAt: number | null;
+  completedAt: number | null;
+  lastError: string | null;
+  cancelRequested: boolean;
+}
+
+const state: IndexingState = {
+  status: 'idle',
+  filesFound: 0,
+  filesProcessed: 0,
+  chunksCreated: 0,
+  currentFile: null,
+  startedAt: null,
+  completedAt: null,
+  lastError: null,
+  cancelRequested: false,
+};
+
+export function getState(): Readonly<IndexingState> {
+  return { ...state };
+}
+
+export function setIndexing(filesFound: number): void {
+  state.status = 'indexing';
+  state.filesFound = filesFound;
+  state.filesProcessed = 0;
+  state.chunksCreated = 0;
+  state.currentFile = null;
+  state.startedAt = Date.now();
+  state.completedAt = null;
+  state.lastError = null;
+  state.cancelRequested = false;
+}
+
+export function updateProgress(filesProcessed: number, chunksCreated: number, currentFile: string): void {
+  state.filesProcessed = filesProcessed;
+  state.chunksCreated = chunksCreated;
+  state.currentFile = currentFile;
+}
+
+export function setReady(stats: { filesProcessed: number; chunksCreated: number }): void {
+  state.status = 'ready';
+  state.filesProcessed = stats.filesProcessed;
+  state.chunksCreated = stats.chunksCreated;
+  state.currentFile = null;
+  state.completedAt = Date.now();
+}
+
+export function setError(msg: string): void {
+  state.status = 'error';
+  state.lastError = msg;
+  state.currentFile = null;
+  state.completedAt = Date.now();
+}
+
+export function requestCancel(): void {
+  state.cancelRequested = true;
+}
+
+export function isCancelRequested(): boolean {
+  return state.cancelRequested;
+}

package/src/indexer/types.ts

@@ -0,0 +1,34 @@
+export interface DocChunk {
+  id: string;
+  filePath: string;
+  fileName: string;
+  category: string;
+  headingHierarchy: string[];
+  headingSlug: string;
+  chunkIndex: number;
+  content: string;
+  charCount: number;
+  lastModified: number;
+}
+
+export interface ChunkMetadata {
+  title?: string;
+  tags?: string[];
+  category?: string;
+}
+
+export interface IndexStats {
+  filesProcessed: number;
+  chunksCreated: number;
+  duration: number;
+}
+
+export interface IndexedFile {
+  filePath: string;
+  lastModified: number;
+  chunkCount: number;
+  title: string | null;
+  category: string | null;
+  indexedAt: number;
+  chunkIds: string | null;
+}

package/src/scripts/diag.ts

@@ -0,0 +1,154 @@
+import { config } from '../config.ts';
+import { initMetadataDb } from '../store/metadata-db.ts';
+import { initStore, getStore, insertChunks } from '../store/orama-store.ts';
+import { search } from '@orama/orama';
+import { embedTexts, embedQuery, checkOllamaHealth } from '../indexer/embedder.ts';
+import { chunkMarkdown } from '../indexer/chunker.ts';
+import path from 'path';
+import { mkdirSync, writeFileSync } from 'fs';
+
+const TEST_DOC = `# Grip System
+
+The grip system allows characters to pick up and hold objects realistically.
+It uses inverse kinematics to adjust hand positions based on the object's shape.
+
+## How It Works
+
+When a character approaches an object, the system calculates grip points
+based on the object's collision mesh. The IK solver then positions the
+character's hands to match these grip points naturally.
+
+## Configuration
+
+You can configure grip strength, release threshold, and IK blend speed
+in the grip component settings.
+`;
+
+async function diag() {
+  console.log('=== Sextant Embedding Diagnostic ===\n');
+
+  // Step 1: Check Ollama
+  console.log('1. Checking Ollama health...');
+  const healthy = await checkOllamaHealth();
+  console.log(`   Ollama: ${healthy ? 'healthy' : 'UNREACHABLE'}`);
+  console.log(`   Model: ${config.embeddingModel}`);
+  console.log(`   Dims: ${config.embeddingDims}`);
+  if (!healthy) {
+    console.log('   STOPPING: Ollama must be running');
+    return;
+  }
+
+  // Step 2: Test raw embedding
+  console.log('\n2. Testing raw embedding call...');
+  try {
+    const embeddings = await embedTexts(['Hello world test']);
+    console.log(`   Got ${embeddings.length} embedding(s)`);
+    console.log(`   Dimensions: ${embeddings[0]?.length}`);
+    console.log(`   First 5 values: [${embeddings[0]?.slice(0, 5).map(v => v.toFixed(6)).join(', ')}]`);
+    const allZero = embeddings[0]?.every(v => v === 0);
+    console.log(`   All zeros: ${allZero}`);
+    if (embeddings[0]?.length !== config.embeddingDims) {
+      console.log(`   WARNING: Model returns ${embeddings[0]?.length} dims but config expects ${config.embeddingDims}`);
+    }
+  } catch (err) {
+    console.log(`   ERROR: ${err}`);
+    return;
+  }
+
+  // Step 3: Create fresh store and index test doc
+  console.log('\n3. Creating fresh store and indexing test doc...');
+  mkdirSync(config.dataPath, { recursive: true });
+  initMetadataDb();
+  await initStore();
+
+  const testDocPath = path.join(config.docsPath, '_diag_test.md');
+  writeFileSync(testDocPath, TEST_DOC);
+
+  const { chunks } = chunkMarkdown(testDocPath, TEST_DOC, Date.now(), config.docsPath);
+  console.log(`   Chunks created: ${chunks.length}`);
+  for (const c of chunks) {
+    console.log(`   - "${c.headingSlug}" (${c.content.length} chars)`);
+  }
+
+  // Step 4: Embed chunks
+  console.log('\n4. Embedding chunks...');
+  const texts = chunks.map(c => c.content);
+  const embeddings = await embedTexts(texts);
+  console.log(`   Got ${embeddings.length} embeddings`);
+  for (let i = 0; i < embeddings.length; i++) {
+    const emb = embeddings[i]!;
+    console.log(`   Chunk ${i}: ${emb.length} dims, first 3: [${emb.slice(0, 3).map(v => v.toFixed(6)).join(', ')}], allZero: ${emb.every(v => v === 0)}`);
+  }
+
+  // Step 5: Insert into store
+  console.log('\n5. Inserting into Orama...');
+  await insertChunks(chunks, embeddings);
+  console.log('   Insert successful');
+
+  // Step 6: Verify with keyword search
+  console.log('\n6. Keyword search for "grip"...');
+  const kwResults = await search(getStore(), { mode: 'fulltext', term: 'grip', limit: 3 } as any);
+  console.log(`   Hits: ${kwResults.hits.length}`);
+  for (const hit of kwResults.hits) {
+    const doc = hit.document as any;
+    console.log(`   - score=${hit.score.toFixed(4)}, section="${doc.headingSlug}"`);
+    const emb = doc.embedding;
+    if (Array.isArray(emb)) {
+      console.log(`     stored embedding: ${emb.length} dims, first 3: [${emb.slice(0, 3).map((v: number) => v.toFixed(6)).join(', ')}]`);
+    } else {
+      console.log(`     stored embedding: ${typeof emb} (NOT an array!)`);
+    }
+  }
+
+  // Step 7: Vector search
+  console.log('\n7. Vector search for "grip system"...');
+  try {
+    const qEmb = await embedQuery('grip system');
+    console.log(`   Query embedding: ${qEmb.length} dims`);
+
+    const vecResults = await search(getStore(), {
+      mode: 'vector',
+      vector: { value: qEmb, property: 'embedding' },
+      similarity: config.similarityThreshold,
+      limit: 3,
+    } as any);
+    console.log(`   Hits: ${vecResults.hits.length}`);
+    for (const hit of vecResults.hits) {
+      const doc = hit.document as any;
+      console.log(`   - score=${hit.score.toFixed(4)}, section="${doc.headingSlug}"`);
+    }
+  } catch (err) {
+    console.log(`   ERROR: ${err}`);
+  }
+
+  // Step 8: Hybrid search
+  console.log('\n8. Hybrid search for "picking up objects"...');
+  try {
+    const qEmb = await embedQuery('picking up objects');
+    const hybridResults = await search(getStore(), {
+      mode: 'hybrid',
+      term: 'picking up objects',
+      vector: { value: qEmb, property: 'embedding' },
+      similarity: config.similarityThreshold,
+      limit: 3,
+      hybridWeights: { text: 0.5, vector: 0.5 },
+    } as any);
+    console.log(`   Hits: ${hybridResults.hits.length}`);
+    for (const hit of hybridResults.hits) {
+      const doc = hit.document as any;
+      console.log(`   - score=${hit.score.toFixed(4)}, section="${doc.headingSlug}"`);
+    }
+  } catch (err) {
+    console.log(`   ERROR: ${err}`);
+  }
+
+  // Cleanup
+  try {
+    const { unlinkSync } = require('fs');
+    unlinkSync(testDocPath);
+  } catch {}
+
+  console.log('\n=== Done ===');
+}
+
+diag().catch(console.error);

package/src/server.ts

@@ -0,0 +1,98 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { z } from 'zod';
+import { handleSearch } from './tools/search.ts';
+import { handleList } from './tools/list.ts';
+import { handleGet } from './tools/get.ts';
+import { handleReindex } from './tools/reindex.ts';
+import { handleStatus } from './tools/status.ts';
+
+export function createMcpServer(): McpServer {
+  const server = new McpServer(
+    { name: 'sextant', version: '0.1.0' },
+    {
+      instructions: 'Sextant provides hybrid semantic and keyword search over project documentation. Use its tools to find information about architecture, decisions, guides, issues, worklogs, plans, and any project knowledge stored in markdown files.',
+    },
+  );
+
+  // search_docs tool
+  server.tool(
+    'search_docs',
+    'Search project documentation using hybrid semantic + keyword search. Returns the most relevant chunks from the docs folder. Use this to find information about architecture, decisions, guides, issues, worklogs, plans, and any project knowledge.',
+    {
+      query: z.string().describe(
+        "Natural language search query OR exact keyword/identifier (e.g., 'how does replication work' or 'ISM-247')"
+      ),
+      top_k: z.number().optional().describe('Number of results to return (default: 10, max: 30)'),
+      category: z.string().optional().describe(
+        "Optional: filter by doc category/folder (e.g., 'architecture', 'worklogs', 'issues')"
+      ),
+      search_mode: z
+        .enum(['hybrid', 'semantic', 'keyword'])
+        .optional()
+        .describe(
+          "Search mode: 'hybrid' (default, best recall), 'semantic' (conceptual similarity only), 'keyword' (exact/token match only)"
+        ),
+    },
+    async (args) => {
+      const result = await handleSearch(args);
+      return { content: [{ type: 'text', text: result }] };
+    }
+  );
+
+  // list_docs tool
+  server.tool(
+    'list_docs',
+    'List all indexed documents, optionally filtered by category. Returns file paths, categories, and document titles.',
+    {
+      category: z.string().optional().describe('Optional: filter by category/folder name'),
+    },
+    async (args) => {
+      const result = handleList(args);
+      return { content: [{ type: 'text', text: result }] };
+    }
+  );
+
+  // get_doc tool
+  server.tool(
+    'get_doc',
+    'Retrieve the full content of a specific document by its file path. Use after search_docs to read a complete document.',
+    {
+      path: z.string().describe(
+        "Relative file path within the docs folder (e.g., 'architecture/networking.md')"
+      ),
+    },
+    async (args) => {
+      const result = await handleGet(args);
+      return { content: [{ type: 'text', text: result }] };
+    }
+  );
+
+  // reindex_docs tool
+  server.tool(
+    'reindex_docs',
+    'Force a full re-index of all documents. Use if search results seem stale or docs have been updated.',
+    {
+      clear_existing: z
+        .boolean()
+        .optional()
+        .describe('If true, wipe all existing index data before re-indexing (default: true)'),
+    },
+    async (args) => {
+      const result = await handleReindex(args);
+      return { content: [{ type: 'text', text: result }] };
+    }
+  );
+
+  // sextant_status tool
+  server.tool(
+    'sextant_status',
+    'Check Sextant health: indexing progress, Ollama connectivity, and index stats.',
+    {},
+    async () => {
+      const result = await handleStatus();
+      return { content: [{ type: 'text', text: result }] };
+    }
+  );
+
+  return server;
+}