@tideshift/sextant 0.1.0

package/src/store/metadata-db.ts

@@ -0,0 +1,109 @@
+import { Database } from 'bun:sqlite';
+import path from 'path';
+import { config } from '../config.ts';
+import type { IndexedFile } from '../indexer/types.ts';
+
+let db: Database | null = null;
+
+export function initMetadataDb(): Database {
+  const dbPath = path.join(config.dataPath, 'metadata.db');
+  db = new Database(dbPath);
+  db.run('PRAGMA journal_mode = WAL');
+  db.run(`
+    CREATE TABLE IF NOT EXISTS indexed_files (
+      filePath TEXT PRIMARY KEY,
+      lastModified INTEGER NOT NULL,
+      chunkCount INTEGER NOT NULL,
+      title TEXT,
+      category TEXT,
+      indexedAt INTEGER NOT NULL
+    )
+  `);
+
+  // Migration: add chunkIds column for deterministic chunk removal
+  const columns = db.query('PRAGMA table_info(indexed_files)').all() as { name: string }[];
+  if (!columns.some((c) => c.name === 'chunkIds')) {
+    db.run('ALTER TABLE indexed_files ADD COLUMN chunkIds TEXT');
+  }
+
+  return db;
+}
+
+export function getMetadataDb(): Database {
+  if (!db) throw new Error('Metadata DB not initialized. Call initMetadataDb() first.');
+  return db;
+}
+
+export function upsertFile(
+  filePath: string,
+  lastModified: number,
+  chunkCount: number,
+  title: string | null,
+  category: string | null,
+  chunkIds?: string[]
+): void {
+  const d = getMetadataDb();
+  d.query(
+    `INSERT OR REPLACE INTO indexed_files (filePath, lastModified, chunkCount, title, category, indexedAt, chunkIds)
+     VALUES ($filePath, $lastModified, $chunkCount, $title, $category, $indexedAt, $chunkIds)`
+  ).run({
+    $filePath: filePath,
+    $lastModified: lastModified,
+    $chunkCount: chunkCount,
+    $title: title,
+    $category: category,
+    $indexedAt: Date.now(),
+    $chunkIds: chunkIds ? JSON.stringify(chunkIds) : null,
+  });
+}
+
+export function getChunkIds(filePath: string): string[] | null {
+  const file = getFile(filePath);
+  if (!file?.chunkIds) return null;
+  try {
+    return JSON.parse(file.chunkIds);
+  } catch {
+    return null;
+  }
+}
+
+export function getFile(filePath: string): IndexedFile | null {
+  const d = getMetadataDb();
+  return d.query('SELECT * FROM indexed_files WHERE filePath = $filePath').get({
+    $filePath: filePath,
+  }) as IndexedFile | null;
+}
+
+export function removeFile(filePath: string): void {
+  const d = getMetadataDb();
+  d.query('DELETE FROM indexed_files WHERE filePath = $filePath').run({
+    $filePath: filePath,
+  });
+}
+
+export function listFiles(category?: string): IndexedFile[] {
+  const d = getMetadataDb();
+  if (category) {
+    return d.query('SELECT * FROM indexed_files WHERE category = $category ORDER BY filePath').all({
+      $category: category,
+    }) as IndexedFile[];
+  }
+  return d.query('SELECT * FROM indexed_files ORDER BY filePath').all() as IndexedFile[];
+}
+
+export function getAllFiles(): IndexedFile[] {
+  const d = getMetadataDb();
+  return d.query('SELECT * FROM indexed_files ORDER BY filePath').all() as IndexedFile[];
+}
+
+export function clearAll(): void {
+  const d = getMetadataDb();
+  d.run('DELETE FROM indexed_files');
+}
+
+export function closeMetadataDb(): void {
+  if (db) {
+    db.close();
+    db = null;
+  }
+}

package/src/store/orama-store.ts

@@ -0,0 +1,172 @@
+import { create, insert, insertMultiple, remove, search, save, load } from '@orama/orama';
+import type { Orama, Results, SearchParams } from '@orama/orama';
+import { config } from '../config.ts';
+import type { DocChunk } from '../indexer/types.ts';
+
+const SCHEMA = {
+  id: 'string' as const,
+  filePath: 'string' as const,
+  fileName: 'string' as const,
+  category: 'string' as const,
+  headingSlug: 'string' as const,
+  content: 'string' as const,
+  chunkIndex: 'number' as const,
+  lastModified: 'number' as const,
+  embedding: `vector[${config.embeddingDims}]` as const,
+};
+
+export type OramaDB = Orama<typeof SCHEMA>;
+
+let db: OramaDB | null = null;
+
+export function getSchema() {
+  return SCHEMA;
+}
+
+export async function initStore(): Promise<OramaDB> {
+  db = await create({ schema: SCHEMA }) as OramaDB;
+  return db;
+}
+
+export function getStore(): OramaDB {
+  if (!db) throw new Error('Orama store not initialized. Call initStore() first.');
+  return db;
+}
+
+export function setStore(newDb: OramaDB): void {
+  db = newDb;
+}
+
+export async function insertChunks(chunks: DocChunk[], embeddings: number[][]): Promise<void> {
+  const store = getStore();
+  const docs = chunks.map((chunk, i) => ({
+    id: chunk.id,
+    filePath: chunk.filePath,
+    fileName: chunk.fileName,
+    category: chunk.category,
+    headingSlug: chunk.headingSlug,
+    content: chunk.content,
+    chunkIndex: chunk.chunkIndex,
+    lastModified: chunk.lastModified,
+    embedding: embeddings[i]!,
+  }));
+
+  await insertMultiple(store, docs);
+}
+
+export async function removeByIds(ids: string[]): Promise<number> {
+  const store = getStore();
+  let removed = 0;
+  for (const id of ids) {
+    try {
+      await remove(store, id);
+      removed++;
+    } catch {
+      // ID not found in store, skip
+    }
+  }
+  return removed;
+}
+
+export async function removeByFile(filePath: string): Promise<number> {
+  const store = getStore();
+  // Fallback: search for chunks by filePath (used when chunkIds are not available)
+  const results = await search(store, {
+    mode: 'fulltext',
+    term: filePath,
+    properties: ['filePath'],
+    limit: 10000,
+  } as any);
+
+  let removed = 0;
+  for (const hit of results.hits) {
+    if ((hit.document as any).filePath === filePath) {
+      await remove(store, hit.id);
+      removed++;
+    }
+  }
+  return removed;
+}
+
+export async function searchHybrid(
+  query: string,
+  queryEmbedding: number[],
+  topK: number,
+  category?: string
+): Promise<Results<any>> {
+  const store = getStore();
+  const params: any = {
+    mode: 'hybrid',
+    term: query,
+    vector: {
+      value: queryEmbedding,
+      property: 'embedding',
+    },
+    similarity: config.similarityThreshold,
+    limit: topK,
+    hybridWeights: {
+      text: config.hybridWeightText,
+      vector: config.hybridWeightVector,
+    },
+  };
+
+  if (category) {
+    params.where = { category: { eq: category } };
+  }
+
+  return search(store, params);
+}
+
+export async function searchKeyword(query: string, topK: number, category?: string): Promise<Results<any>> {
+  const store = getStore();
+  const params: any = {
+    mode: 'fulltext',
+    term: query,
+    limit: topK,
+  };
+
+  if (category) {
+    params.where = { category: { eq: category } };
+  }
+
+  return search(store, params);
+}
+
+export async function searchVector(queryEmbedding: number[], topK: number, category?: string): Promise<Results<any>> {
+  const store = getStore();
+  const params: any = {
+    mode: 'vector',
+    vector: {
+      value: queryEmbedding,
+      property: 'embedding',
+    },
+    similarity: config.similarityThreshold,
+    limit: topK,
+  };
+
+  if (category) {
+    params.where = { category: { eq: category } };
+  }
+
+  return search(store, params);
+}
+
+export async function getDocumentCount(): Promise<number> {
+  const store = getStore();
+  const results = await search(store, {
+    mode: 'fulltext',
+    term: '',
+    limit: 0,
+  } as any);
+  return results.count;
+}
+
+export async function saveIndex(): Promise<any> {
+  return save(getStore());
+}
+
+export async function loadIndex(data: any): Promise<void> {
+  const store = await create({ schema: SCHEMA }) as OramaDB;
+  await load(store, data);
+  db = store;
+}

package/src/store/persistence.ts

@@ -0,0 +1,99 @@
+import path from 'path';
+import { rename } from 'fs/promises';
+import { config } from '../config.ts';
+import { saveIndex, loadIndex, initStore, getStore } from './orama-store.ts';
+import { search } from '@orama/orama';
+import { clearAll as clearMetadata } from './metadata-db.ts';
+
+const ORAMA_PATH = path.join(config.dataPath, 'orama.bin');
+
+let persistTimer: ReturnType<typeof setTimeout> | null = null;
+let lastKnownMtimeMs = 0;
+
+export async function persistToDisk(): Promise<void> {
+  try {
+    const data = await saveIndex();
+    // Atomic write: write to temp file, then rename to avoid torn reads by concurrent instances
+    const tempPath = ORAMA_PATH + '.tmp';
+    await Bun.write(tempPath, JSON.stringify(data));
+    await rename(tempPath, ORAMA_PATH);
+    // Track mtime so we know this write is ours
+    const stat = await Bun.file(ORAMA_PATH).stat();
+    if (stat) lastKnownMtimeMs = stat.mtimeMs;
+    console.error('[persistence] Index saved to disk');
+  } catch (err) {
+    console.error('[persistence] Failed to save index:', err);
+  }
+}
+
+export function debouncedPersist(): void {
+  if (persistTimer) clearTimeout(persistTimer);
+  persistTimer = setTimeout(() => {
+    persistToDisk().catch((err) => console.error('[persistence] Debounced persist failed:', err));
+  }, config.persistDebounceMs);
+}
+
+export async function loadFromDisk(): Promise<boolean> {
+  try {
+    const file = Bun.file(ORAMA_PATH);
+    if (await file.exists()) {
+      const data = JSON.parse(await file.text());
+      await loadIndex(data);
+
+      // Validate that the stored index dimensions match the current config
+      const probe = new Float32Array(config.embeddingDims);
+      try {
+        await search(getStore(), {
+          mode: 'vector',
+          vector: { value: Array.from(probe), property: 'embedding' },
+          limit: 1,
+        } as any);
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : '';
+        if (msg.includes('dimensional')) {
+          console.error(`[persistence] Dimension mismatch in stored index, discarding stale data`);
+          await initStore();
+          clearMetadata();
+          return false;
+        }
+        // Other errors are fine (e.g. empty index)
+      }
+
+      const stat = await file.stat();
+      if (stat) lastKnownMtimeMs = stat.mtimeMs;
+      console.error('[persistence] Index loaded from disk');
+      return true;
+    }
+  } catch (err) {
+    console.error('[persistence] Failed to load index from disk, starting fresh:', err);
+  }
+  return false;
+}
+
+export async function reloadIfChanged(): Promise<boolean> {
+  try {
+    const file = Bun.file(ORAMA_PATH);
+    if (!(await file.exists())) return false;
+
+    const stat = await file.stat();
+    if (!stat || stat.mtimeMs <= lastKnownMtimeMs) return false;
+
+    console.error('[persistence] Index file changed on disk, reloading...');
+    const data = JSON.parse(await file.text());
+    await loadIndex(data);
+    lastKnownMtimeMs = stat.mtimeMs;
+    console.error('[persistence] Index reloaded from disk');
+    return true;
+  } catch (err) {
+    console.error('[persistence] Failed to reload index:', err);
+    return false;
+  }
+}
+
+export async function flushPersist(): Promise<void> {
+  if (persistTimer) {
+    clearTimeout(persistTimer);
+    persistTimer = null;
+  }
+  await persistToDisk();
+}

package/src/tools/get.ts

@@ -0,0 +1,22 @@
+import path from 'path';
+import { config } from '../config.ts';
+
+interface GetArgs {
+  path: string;
+}
+
+export async function handleGet(args: GetArgs): Promise<string> {
+  const filePath = path.join(config.docsPath, args.path);
+
+  try {
+    const file = Bun.file(filePath);
+    if (!(await file.exists())) {
+      return `Error: File not found: ${args.path}`;
+    }
+    const content = await file.text();
+    return `# ${args.path}\n\n${content}`;
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return `Error reading file: ${msg}`;
+  }
+}

package/src/tools/list.ts

@@ -0,0 +1,31 @@
+import { listFiles } from '../store/metadata-db.ts';
+import { getState } from '../indexer/state.ts';
+
+interface ListArgs {
+  category?: string;
+}
+
+export function handleList(args: ListArgs): string {
+  const files = listFiles(args.category);
+
+  if (files.length === 0) {
+    const indexState = getState();
+    if (indexState.status === 'indexing') {
+      return `Sextant is still performing initial indexing (${indexState.filesProcessed}/${indexState.filesFound} files). Document list will be available shortly. Use sextant_status to check progress.`;
+    }
+    return args.category
+      ? `No documents found in category "${args.category}".`
+      : 'No documents indexed yet.';
+  }
+
+  const lines: string[] = [
+    `Indexed documents${args.category ? ` (category: ${args.category})` : ''}: ${files.length} files`,
+    '',
+  ];
+
+  for (const f of files) {
+    lines.push(`- ${f.filePath} [${f.category}] (${f.chunkCount} chunks)${f.title ? ` — ${f.title}` : ''}`);
+  }
+
+  return lines.join('\n');
+}

package/src/tools/reindex.ts

@@ -0,0 +1,29 @@
+import { config } from '../config.ts';
+import { fullReindex } from '../indexer/pipeline.ts';
+import { getState } from '../indexer/state.ts';
+
+interface ReindexArgs {
+  clear_existing?: boolean;
+}
+
+export async function handleReindex(args: ReindexArgs): Promise<string> {
+  const indexState = getState();
+  if (indexState.status === 'indexing') {
+    return `Indexing is already in progress (${indexState.filesProcessed}/${indexState.filesFound} files). Use sextant_status to monitor progress.`;
+  }
+
+  const clearExisting = args.clear_existing !== false; // default true
+
+  console.error(`[reindex] Starting full reindex (clear_existing: ${clearExisting})...`);
+
+  // Fire and forget -- return immediately
+  fullReindex(config.docsPath, clearExisting)
+    .then((stats) => {
+      console.error(`[reindex] Complete: ${stats.filesProcessed} files, ${stats.chunksCreated} chunks in ${stats.duration}ms`);
+    })
+    .catch((err) => {
+      console.error('[reindex] Failed:', err);
+    });
+
+  return `Reindex started. Use sextant_status to monitor progress.`;
+}

package/src/tools/search.ts

@@ -0,0 +1,93 @@
+import { config } from '../config.ts';
+import { embedQuery, checkOllamaHealth } from '../indexer/embedder.ts';
+import { getState } from '../indexer/state.ts';
+import { checkAndReindex } from '../indexer/freshness.ts';
+import { searchHybrid, searchKeyword, searchVector } from '../store/orama-store.ts';
+import { listFiles } from '../store/metadata-db.ts';
+
+interface SearchArgs {
+  query: string;
+  top_k?: number;
+  category?: string;
+  search_mode?: 'hybrid' | 'semantic' | 'keyword';
+}
+
+export async function handleSearch(args: SearchArgs): Promise<string> {
+  const { query, category, search_mode = 'hybrid' } = args;
+  const topK = Math.min(args.top_k ?? config.defaultTopK, 30);
+
+  // Check for stale docs and trigger background reindex if needed
+  const freshness = await checkAndReindex();
+
+  const indexState = getState();
+  let statusPrefix = '';
+
+  if (freshness.stale && !freshness.alreadyIndexing) {
+    statusPrefix = `Note: ${freshness.staleCount} file(s) changed since last index. Background reindex started. Results may not reflect latest changes.\n\n`;
+  }
+
+  if (!statusPrefix) {
+    if (indexState.status === 'indexing') {
+      const docCount = listFiles().length;
+      if (docCount === 0) {
+        return `Sextant is still performing initial indexing (${indexState.filesProcessed}/${indexState.filesFound} files). Search will be available shortly. Use sextant_status to check progress.`;
+      }
+      statusPrefix = `Note: Indexing is in progress (${indexState.filesProcessed}/${indexState.filesFound} files). Results may be incomplete.\n\n`;
+    } else if (indexState.status === 'error') {
+      statusPrefix = `Warning: Last indexing failed: ${indexState.lastError}\n\n`;
+    }
+  }
+
+  try {
+    let results;
+
+    if (search_mode === 'keyword') {
+      results = await searchKeyword(query, topK, category);
+    } else {
+      // Hybrid or semantic both need embeddings
+      const ollamaOk = await checkOllamaHealth();
+      if (!ollamaOk) {
+        if (search_mode === 'semantic') {
+          return 'Error: Embedding service unavailable — start Ollama with `ollama serve` and pull `qwen3-embedding:0.6b`. Semantic search requires embeddings.';
+        }
+        // Fall back to keyword for hybrid
+        console.error('[search] Ollama unavailable, falling back to keyword search');
+        results = await searchKeyword(query, topK, category);
+      } else {
+        const queryEmbedding = await embedQuery(query);
+
+        if (search_mode === 'semantic') {
+          results = await searchVector(queryEmbedding, topK, category);
+        } else {
+          results = await searchHybrid(query, queryEmbedding, topK, category);
+        }
+      }
+    }
+
+    if (results.hits.length === 0) {
+      return statusPrefix + `No results found for "${query}"${category ? ` in category "${category}"` : ''}.`;
+    }
+
+    const lines: string[] = [
+      `Found ${results.hits.length} results for "${query}" (mode: ${search_mode}):`,
+      '',
+    ];
+
+    for (let i = 0; i < results.hits.length; i++) {
+      const hit = results.hits[i]!;
+      const doc = hit.document as any;
+      lines.push(`--- Result ${i + 1} (score: ${hit.score.toFixed(4)}) ---`);
+      lines.push(`File: ${doc.filePath}`);
+      lines.push(`Section: ${doc.headingSlug}`);
+      lines.push(`Category: ${doc.category}`);
+      lines.push('');
+      lines.push(doc.content);
+      lines.push('');
+    }
+
+    return statusPrefix + lines.join('\n');
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return `Search error: ${msg}`;
+  }
+}

package/src/tools/status.ts

@@ -0,0 +1,53 @@
+import { config } from '../config.ts';
+import { getState } from '../indexer/state.ts';
+import { checkOllamaHealth } from '../indexer/embedder.ts';
+import { listFiles } from '../store/metadata-db.ts';
+
+export async function handleStatus(): Promise<string> {
+  const state = getState();
+  const ollamaOk = await checkOllamaHealth();
+  const files = listFiles();
+
+  const lines: string[] = ['Sextant Status'];
+
+  // Indexing status
+  if (state.status === 'indexing') {
+    lines.push(`  Indexing:  in progress (${state.filesProcessed}/${state.filesFound} files${state.currentFile ? `, current: ${state.currentFile}` : ''})`);
+  } else if (state.status === 'ready') {
+    const duration = state.startedAt && state.completedAt
+      ? ((state.completedAt - state.startedAt) / 1000).toFixed(1)
+      : '?';
+    lines.push(`  Indexing:  ready (${state.filesProcessed} files, ${state.chunksCreated} chunks, took ${duration}s)`);
+  } else if (state.status === 'error') {
+    lines.push(`  Indexing:  error: ${state.lastError}`);
+  } else {
+    lines.push(`  Indexing:  idle`);
+  }
+
+  // Ollama health
+  if (ollamaOk) {
+    lines.push(`  Ollama:    healthy (${config.embeddingModel} at ${config.ollamaUrl})`);
+  } else {
+    lines.push(`  Ollama:    unreachable at ${config.ollamaUrl}`);
+  }
+
+  // Index stats grouped by category
+  if (files.length > 0) {
+    const byCategory = new Map<string, number>();
+    for (const f of files) {
+      const cat = f.category ?? 'root';
+      byCategory.set(cat, (byCategory.get(cat) ?? 0) + 1);
+    }
+    lines.push(`  Index:     ${files.length} files across ${byCategory.size} categories`);
+    for (const [cat, count] of [...byCategory.entries()].sort()) {
+      lines.push(`               ${cat}: ${count} files`);
+    }
+  } else {
+    lines.push(`  Index:     empty`);
+  }
+
+  // Config
+  lines.push(`  Docs path: ${config.docsPath}`);
+
+  return lines.join('\n');
+}