@geminilight/mindos 0.6.67 → 0.6.70

package/app/lib/acp/types.ts

@@ -100,37 +100,8 @@ export interface AcpSessionInfo {
   updatedAt?: string;
 }
 
-/* ── JSON-RPC (ACP uses JSON-RPC 2.0 over stdio) ─────────────────────── */
-
-export interface AcpJsonRpcRequest {
-  jsonrpc: '2.0';
-  id: string | number;
-  method: string;
-  params?: Record<string, unknown>;
-}
-
-export interface AcpJsonRpcResponse {
-  jsonrpc: '2.0';
-  id: string | number | null;
-  result?: unknown;
-  error?: AcpJsonRpcError;
-}
-
-export interface AcpJsonRpcError {
-  code: number;
-  message: string;
-  data?: unknown;
-}
-
 /* ── Prompt ────────────────────────────────────────────────────────────── */
 
-export interface AcpPromptRequest {
-  sessionId: string;
-  prompt: AcpContentBlock[];
-  context?: { cwd?: string };
-  stream?: boolean;
-}
-
 export interface AcpPromptResponse {
   sessionId: string;
   text: string;

package/app/lib/agent/model.ts

@@ -13,6 +13,18 @@ function ensureVisionCapable(model: Model<any>): Model<any> {
   return { ...model, input: [...inputs, 'image'] as any };
 }
 
+/**
+ * Normalize a user-provided baseUrl without changing its semantic path.
+ *
+ * We only trim whitespace and trailing slashes. We intentionally do not
+ * rewrite path segments like `/1` or `/v1`, because some gateways may use
+ * custom prefixes and mutating them could break valid configurations.
+ */
+export function normalizeBaseUrl(url: string): string {
+  if (!url) return url;
+  return url.trim().replace(/\/+$/, '');
+}
+
 export interface ModelConfigOverrides {
   provider?: ProviderId;
   apiKey?: string;
@@ -44,13 +56,14 @@ export function getModelConfig(options?: ModelConfigOverrides): {
   };
 
   const modelName = cfg.model;
-  let model = resolveModel(cfg.provider, modelName, cfg.baseUrl);
+  const normalizedBaseUrl = normalizeBaseUrl(cfg.baseUrl);
+  let model = resolveModel(cfg.provider, modelName, normalizedBaseUrl);
 
   if (options?.hasImages) {
     model = ensureVisionCapable(model);
   }
 
-  return { model, modelName, apiKey: cfg.apiKey, provider: cfg.provider, baseUrl: cfg.baseUrl };
+  return { model, modelName, apiKey: cfg.apiKey, provider: cfg.provider, baseUrl: normalizedBaseUrl };
 }
 
 /**
@@ -61,7 +74,10 @@ function resolveModel(providerId: ProviderId, modelName: string, baseUrl: string
   const piProvider = toPiProvider(providerId);
   const preset = getPreset(providerId);
   let model: Model<any>;
-  const hasCustomBase = !!baseUrl;
+
+  // Normalize user-provided baseUrl before use
+  const normalizedBaseUrl = normalizeBaseUrl(baseUrl);
+  const hasCustomBase = !!normalizedBaseUrl;
 
   // 1. Try pi-ai registry lookup
   try {
@@ -92,7 +108,7 @@ function resolveModel(providerId: ProviderId, modelName: string, baseUrl: string
 
   // 3. Apply user's custom baseUrl
   if (hasCustomBase) {
-    model = { ...model, baseUrl };
+    model = { ...model, baseUrl: normalizedBaseUrl };
 
     if (model.api === 'openai-responses') {
       model = { ...model, api: 'openai-completions' as any };

package/app/lib/agent/non-streaming.ts

@@ -1,5 +1,6 @@
 import type { AgentTool } from '@mariozechner/pi-agent-core';
 import type { MindOSSSEvent } from '@/lib/sse/events';
+import { buildCompatEndpointCandidates } from './providers';
 
 /**
  * Reassemble SSE chunks (from proxies that ignore stream:false) into a
@@ -173,32 +174,48 @@ export async function runNonStreamingFallback(opts: NonStreamingOptions): Promis
   ];
 
   const toolMap = new Map(tools.map(t => [t.name, t]));
-  const endpoint = `${baseUrl.replace(/\/$/, '')}/chat/completions`;
+  const endpoints = buildCompatEndpointCandidates(baseUrl, '/chat/completions', 'openai-completions');
+  const attemptedEndpoints = endpoints.join(' , ');
   let step = 0;
 
   while (step < maxSteps) {
     if (signal.aborted) throw new Error('Request aborted');
     step++;
 
-    const resp = await fetch(endpoint, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-        'Authorization': `Bearer ${apiKey}`,
-      },
-      body: JSON.stringify({
-        model,
-        messages,
-        tools: openaiTools.length > 0 ? openaiTools : undefined,
-        tool_choice: openaiTools.length > 0 ? 'auto' : undefined,
-        stream: true,
-      }),
-      signal,
-    });
-
-    if (!resp.ok) {
-      const errText = await resp.text().catch(() => '');
-      throw new Error(`Non-streaming API error ${resp.status}: ${errText.slice(0, 200)}`);
+    let resp: Response | null = null;
+    let lastEndpointError = '';
+
+    for (const endpoint of endpoints) {
+      const attempt = await fetch(endpoint, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': `Bearer ${apiKey}`,
+        },
+        body: JSON.stringify({
+          model,
+          messages,
+          tools: openaiTools.length > 0 ? openaiTools : undefined,
+          tool_choice: openaiTools.length > 0 ? 'auto' : undefined,
+          stream: true,
+        }),
+        signal,
+      });
+
+      if (attempt.ok) {
+        resp = attempt;
+        break;
+      }
+
+      const errText = await attempt.text().catch(() => '');
+      lastEndpointError = `HTTP ${attempt.status} @ ${endpoint}: ${errText.slice(0, 200)}`;
+      if (attempt.status !== 404) {
+        throw new Error(`Non-streaming API error ${lastEndpointError}`);
+      }
+    }
+
+    if (!resp) {
+      throw new Error(`Non-streaming API error ${lastEndpointError || 'all endpoint candidates failed'}; tried ${attemptedEndpoints}`);
     }
 
     const rawText = await resp.text();

package/app/lib/agent/providers.ts

@@ -323,6 +323,37 @@ export function getDefaultApi(id: ProviderId): string {
   }
 }
 
+/** Return the effective API type used by a provider (openai-completions, anthropic-messages, etc.). */
+export function getProviderApiType(id: ProviderId): string {
+  return getDefaultApi(id);
+}
+
+/**
+ * Build endpoint candidates for compatible providers.
+ *
+ * We do not assume whether the user-supplied baseUrl already includes a version
+ * prefix. For OpenAI- and Anthropic-compatible gateways we try both forms:
+ *   1. base + path
+ *   2. base + /v1 + path (if base does not already contain /vN)
+ *
+ * This avoids breaking providers like GLM/Kimi/MiniMax that may use custom path
+ * prefixes, while still handling common OpenAI-compatible gateways that expect /v1.
+ */
+export function buildCompatEndpointCandidates(baseUrl: string, path: string, apiType: string): string[] {
+  const base = baseUrl.replace(/\/+$/, '');
+  const cleanPath = path.startsWith('/') ? path : `/${path}`;
+  const hasVersionPrefix = /\/v\d+(?:$|\/)/.test(base);
+  const candidates = new Set<string>();
+
+  candidates.add(`${base}${cleanPath}`);
+
+  if (!hasVersionPrefix && (apiType === 'openai-completions' || apiType === 'openai-responses' || apiType === 'anthropic-messages')) {
+    candidates.add(`${base}/v1${cleanPath}`);
+  }
+
+  return Array.from(candidates);
+}
+
 // ---------------------------------------------------------------------------
 // Internal: reverse-engineer pi-ai's env var name mapping (for UI display)
 // ---------------------------------------------------------------------------

package/app/lib/agent/tools.ts

@@ -2,11 +2,13 @@ import path from 'path';
 import { Type, type Static } from '@sinclair/typebox';
 import type { AgentTool, AgentToolResult } from '@mariozechner/pi-agent-core';
 import {
-  searchFiles, getFileContent, getFileTree, getRecentlyModified,
+  getFileContent, getFileTree, getRecentlyModified,
   saveFileContent, createFile, appendToFile, insertAfterHeading, updateSection,
   moveToTrashFile, renameFile, moveFile, findBacklinks, gitLog, gitShowFile, appendCsvRow,
   getMindRoot,
 } from '@/lib/fs';
+import { searchFiles } from '@/lib/core/search';
+import { hybridSearch } from '@/lib/core/hybrid-search';
 import { readSkillContentByName } from '@/lib/pi-integration/skills';
 import { a2aTools } from '@/lib/a2a/a2a-tools';
 import { acpTools } from '@/lib/acp/acp-tools';
@@ -353,9 +355,9 @@ export const knowledgeBaseTools: AgentTool<any>[] = [
     description: 'Full-text search across all files in the knowledge base. Returns matching files with context snippets.',
     parameters: QueryParam,
     execute: safeExecute(async (_id, params: Static<typeof QueryParam>) => {
-      const results = searchFiles(params.query);
+      const results = await hybridSearch(getMindRoot(), params.query);
       if (results.length === 0) return textResult('No results found.');
-      return textResult(results.map(r => `- **${r.path}**: ${r.snippet}`).join('\n'));
+      return textResult(results.map(r => `- **${r.path}** (score: ${r.score.toFixed(1)}): ${r.snippet}`).join('\n'));
     }),
   },
 

package/app/lib/core/embedding-index.ts

@@ -0,0 +1,285 @@
+/**
+ * In-memory embedding vector index with JSON file persistence.
+ *
+ * Stores one embedding vector per file. Supports:
+ * - Cosine similarity search
+ * - Incremental add/update/remove
+ * - Persist to / load from ~/.mindos/embedding-index.json
+ *
+ * Designed for <2000 files — brute-force cosine is fast enough (~2ms for 1000 docs).
+ */
+
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { collectAllFiles } from './tree';
+import { readFile } from './fs-ops';
+import { getEmbeddings, getEmbedding, getEmbeddingConfig } from './embedding-provider';
+
+const MAX_CONTENT_LENGTH = 8_000; // Truncate long files (embedding context window)
+
+export interface EmbeddingSearchResult {
+  path: string;
+  similarity: number; // 0..1 cosine similarity
+}
+
+export class EmbeddingIndex {
+  /** filePath → Float32Array (embedding vector) */
+  private vectors = new Map<string, Float32Array>();
+  private dimensions = 0;
+  private builtForRoot: string | null = null;
+
+  /** Whether a full build or load has been done for this root. */
+  private _ready = false;
+
+  /** Background build in progress — callers should not wait, just use BM25. */
+  private _building = false;
+
+  isReady(): boolean { return this._ready; }
+  isBuilding(): boolean { return this._building; }
+
+  /** Number of indexed documents. */
+  getDocCount(): number { return this.vectors.size; }
+
+  // ── Build ────────────────────────────────────────────────────────
+
+  /**
+   * Full rebuild: embed all files. Runs async — returns immediately.
+   * Callers should check isReady() before calling search().
+   */
+  async rebuild(mindRoot: string): Promise<void> {
+    if (this._building) return;
+    this._building = true;
+
+    try {
+      const config = getEmbeddingConfig();
+      if (!config) { this._building = false; return; }
+
+      const allFiles = collectAllFiles(mindRoot);
+      const texts: string[] = [];
+      const paths: string[] = [];
+
+      for (const filePath of allFiles) {
+        // Skip non-text files for embedding
+        const ext = path.extname(filePath).toLowerCase();
+        if (ext !== '.md' && ext !== '.csv') continue;
+
+        let content: string;
+        try { content = readFile(mindRoot, filePath); } catch { continue; }
+
+        // Prepend file path as context
+        const text = `${filePath}\n${content}`.slice(0, MAX_CONTENT_LENGTH);
+        texts.push(text);
+        paths.push(filePath);
+      }
+
+      if (texts.length === 0) {
+        this._building = false;
+        this._ready = true;
+        this.builtForRoot = mindRoot;
+        return;
+      }
+
+      // Batch embed all documents
+      const vectors = await getEmbeddings(texts);
+
+      if (vectors.length !== texts.length) {
+        console.error(`[embedding-index] Expected ${texts.length} vectors, got ${vectors.length}`);
+        this._building = false;
+        return;
+      }
+
+      this.vectors.clear();
+      this.dimensions = vectors[0].length;
+      for (let i = 0; i < paths.length; i++) {
+        this.vectors.set(paths[i], vectors[i]);
+      }
+
+      this.builtForRoot = mindRoot;
+      this._ready = true;
+
+      // Persist in background
+      try { this.persist(); } catch { /* non-critical */ }
+    } catch (err) {
+      console.error('[embedding-index] Rebuild failed:', err instanceof Error ? err.message : err);
+    } finally {
+      this._building = false;
+    }
+  }
+
+  // ── Incremental updates ──────────────────────────────────────────
+
+  /** Add or update a single file's embedding. */
+  async updateFile(mindRoot: string, filePath: string): Promise<void> {
+    if (!this._ready) return;
+    const ext = path.extname(filePath).toLowerCase();
+    if (ext !== '.md' && ext !== '.csv') return;
+
+    let content: string;
+    try { content = readFile(mindRoot, filePath); } catch { return; }
+
+    const text = `${filePath}\n${content}`.slice(0, MAX_CONTENT_LENGTH);
+    const vector = await getEmbedding(text);
+    if (vector) {
+      this.vectors.set(filePath, vector);
+      if (this.dimensions === 0) this.dimensions = vector.length;
+    }
+  }
+
+  /** Remove a file from the index. */
+  removeFile(filePath: string): void {
+    this.vectors.delete(filePath);
+  }
+
+  // ── Search ───────────────────────────────────────────────────────
+
+  /**
+   * Find the top-K most similar documents to the query vector.
+   * Returns results sorted by descending similarity.
+   */
+  searchByVector(queryVector: Float32Array, topK: number = 20): EmbeddingSearchResult[] {
+    if (!this._ready || this.vectors.size === 0) return [];
+
+    const results: EmbeddingSearchResult[] = [];
+
+    for (const [filePath, docVector] of this.vectors) {
+      const sim = cosineSimilarity(queryVector, docVector);
+      results.push({ path: filePath, similarity: sim });
+    }
+
+    results.sort((a, b) => b.similarity - a.similarity);
+    return results.slice(0, topK);
+  }
+
+  /**
+   * Search by text query: embed the query, then find similar docs.
+   * Returns empty array if embedding fails or index not ready.
+   */
+  async search(query: string, topK: number = 20): Promise<EmbeddingSearchResult[]> {
+    if (!this._ready) return [];
+
+    const queryVector = await getEmbedding(query);
+    if (!queryVector) return [];
+
+    return this.searchByVector(queryVector, topK);
+  }
+
+  // ── Persistence ──────────────────────────────────────────────────
+
+  private get persistPath(): string {
+    return path.join(os.homedir(), '.mindos', 'embedding-index.json');
+  }
+
+  /** Serialize to disk. */
+  persist(): void {
+    if (this.vectors.size === 0) return;
+
+    const data: PersistedEmbeddingIndex = {
+      version: 1,
+      builtForRoot: this.builtForRoot ?? '',
+      dimensions: this.dimensions,
+      docCount: this.vectors.size,
+      timestamp: Date.now(),
+      vectors: {},
+    };
+
+    for (const [filePath, vec] of this.vectors) {
+      // Store as regular number array for JSON serialization
+      data.vectors[filePath] = Array.from(vec);
+    }
+
+    const dir = path.dirname(this.persistPath);
+    try {
+      fs.mkdirSync(dir, { recursive: true });
+      fs.writeFileSync(this.persistPath, JSON.stringify(data), 'utf-8');
+    } catch {
+      // Non-critical
+    }
+  }
+
+  /**
+   * Load from disk. Returns true if loaded successfully.
+   * Staleness check: file count must match and root must match.
+   */
+  load(mindRoot: string): boolean {
+    try {
+      const raw = fs.readFileSync(this.persistPath, 'utf-8');
+      const data: PersistedEmbeddingIndex = JSON.parse(raw);
+
+      if (data.version !== 1 || data.builtForRoot !== mindRoot) return false;
+
+      // Basic staleness: check file count
+      const currentFiles = collectAllFiles(mindRoot);
+      const mdCsvCount = currentFiles.filter(f => {
+        const ext = path.extname(f).toLowerCase();
+        return ext === '.md' || ext === '.csv';
+      }).length;
+
+      // Allow some drift (files added/removed since last persist)
+      // If >10% drift, force rebuild
+      if (Math.abs(mdCsvCount - data.docCount) > Math.max(5, mdCsvCount * 0.1)) {
+        return false;
+      }
+
+      this.vectors.clear();
+      this.dimensions = data.dimensions;
+      for (const [filePath, arr] of Object.entries(data.vectors)) {
+        this.vectors.set(filePath, new Float32Array(arr));
+      }
+      this.builtForRoot = data.builtForRoot;
+      this._ready = true;
+
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /** Clear all state. */
+  invalidate(): void {
+    this.vectors.clear();
+    this.dimensions = 0;
+    this.builtForRoot = null;
+    this._ready = false;
+  }
+
+  /** Check if index is built for this root. */
+  isBuiltFor(mindRoot: string): boolean {
+    return this._ready && this.builtForRoot === mindRoot;
+  }
+}
+
+// ── Math ─────────────────────────────────────────────────────────
+
+/** Cosine similarity between two vectors. Returns 0..1 (clamped). */
+export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+  if (a.length !== b.length || a.length === 0) return 0;
+
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  if (denom === 0) return 0;
+
+  // Clamp to [0, 1] — embeddings from the same model are usually positive,
+  // but numerical noise can push slightly below 0 or above 1.
+  return Math.max(0, Math.min(1, dot / denom));
+}
+
+// ── Persistence types ────────────────────────────────────────────
+
+interface PersistedEmbeddingIndex {
+  version: number;
+  builtForRoot: string;
+  dimensions: number;
+  docCount: number;
+  timestamp: number;
+  vectors: Record<string, number[]>;
+}

package/app/lib/core/embedding-provider.ts

@@ -0,0 +1,164 @@
+/**
+ * Embedding API provider — calls OpenAI-compatible /v1/embeddings endpoint.
+ *
+ * Supports: OpenAI, DeepSeek, Ollama, any OpenAI-compatible embedding API.
+ * Config is stored in ~/.mindos/config.json under the `embedding` key.
+ */
+
+import { readSettings } from '@/lib/settings';
+
+export interface EmbeddingConfig {
+  enabled: boolean;
+  baseUrl: string;   // e.g. "https://api.openai.com/v1"
+  apiKey: string;
+  model: string;     // e.g. "text-embedding-3-small"
+}
+
+/** Read embedding config from settings. Returns null if not configured or disabled. */
+export function getEmbeddingConfig(): EmbeddingConfig | null {
+  try {
+    const s = readSettings();
+    const e = s.embedding;
+    if (!e || !e.enabled) return null;
+    if (!e.baseUrl || !e.model) return null;
+    // apiKey can be empty for local providers like Ollama
+    return {
+      enabled: true,
+      baseUrl: e.baseUrl.replace(/\/+$/, ''), // strip trailing slash
+      apiKey: e.apiKey || '',
+      model: e.model,
+    };
+  } catch {
+    return null;
+  }
+}
+
+/** Maximum texts per batch (OpenAI limit is 2048, we use a conservative default). */
+const BATCH_SIZE = 100;
+
+/** Request timeout in ms. */
+const TIMEOUT_MS = 30_000;
+
+/** Max retries on transient errors (429, 5xx). */
+const MAX_RETRIES = 2;
+
+/**
+ * Get embeddings for an array of texts.
+ * Returns Float32Array[] with one vector per input text.
+ * Returns empty array on failure (graceful fallback — never throws).
+ */
+export async function getEmbeddings(texts: string[]): Promise<Float32Array[]> {
+  const config = getEmbeddingConfig();
+  if (!config || texts.length === 0) return [];
+
+  try {
+    const results: Float32Array[] = [];
+
+    // Process in batches
+    for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+      const batch = texts.slice(i, i + BATCH_SIZE);
+      const batchResults = await callEmbeddingApi(config, batch);
+      if (batchResults.length === 0) return []; // Abort on any batch failure
+      results.push(...batchResults);
+    }
+
+    return results;
+  } catch (err) {
+    console.error('[embedding] Failed to get embeddings:', err instanceof Error ? err.message : err);
+    return [];
+  }
+}
+
+/**
+ * Get embedding for a single text. Convenience wrapper.
+ * Returns null on failure.
+ */
+export async function getEmbedding(text: string): Promise<Float32Array | null> {
+  const results = await getEmbeddings([text]);
+  return results.length > 0 ? results[0] : null;
+}
+
+/** Call the OpenAI-compatible /v1/embeddings endpoint with retry logic. */
+async function callEmbeddingApi(
+  config: EmbeddingConfig,
+  texts: string[],
+): Promise<Float32Array[]> {
+  const url = `${config.baseUrl}/embeddings`;
+  const body = JSON.stringify({
+    model: config.model,
+    input: texts,
+  });
+
+  const headers: Record<string, string> = {
+    'Content-Type': 'application/json',
+  };
+  if (config.apiKey) {
+    headers['Authorization'] = `Bearer ${config.apiKey}`;
+  }
+
+  for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
+
+      const res = await fetch(url, {
+        method: 'POST',
+        headers,
+        body,
+        signal: controller.signal,
+      });
+
+      clearTimeout(timer);
+
+      if (!res.ok) {
+        const status = res.status;
+        // Retry on rate limit or server errors
+        if ((status === 429 || status >= 500) && attempt < MAX_RETRIES) {
+          const retryAfter = parseInt(res.headers.get('retry-after') || '1', 10);
+          await sleep(Math.min(retryAfter * 1000, 5000));
+          continue;
+        }
+        const errText = await res.text().catch(() => '');
+        console.error(`[embedding] API error ${status}: ${errText.slice(0, 200)}`);
+        return [];
+      }
+
+      const json = await res.json() as EmbeddingResponse;
+      if (!json.data || !Array.isArray(json.data)) {
+        console.error('[embedding] Unexpected response shape');
+        return [];
+      }
+
+      // Sort by index (API may return out of order)
+      json.data.sort((a, b) => a.index - b.index);
+
+      return json.data.map(d => new Float32Array(d.embedding));
+    } catch (err) {
+      if (attempt < MAX_RETRIES) {
+        await sleep(1000 * (attempt + 1)); // Exponential-ish backoff
+        continue;
+      }
+      console.error('[embedding] Request failed:', err instanceof Error ? err.message : err);
+      return [];
+    }
+  }
+
+  return [];
+}
+
+/** OpenAI /v1/embeddings response shape. */
+interface EmbeddingResponse {
+  data: Array<{
+    index: number;
+    embedding: number[];
+  }>;
+  model: string;
+  usage?: {
+    prompt_tokens: number;
+    total_tokens: number;
+  };
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}