memorix 1.2.2 → 1.2.4

package/src/compact/token-budget.ts

@@ -1,74 +1,89 @@
-/**
- * Token Budget Manager
- *
- * Provides token counting and budget management for Progressive Disclosure.
- * Source: gpt-tokenizer (737 stars, JS port of OpenAI's tiktoken)
- *
- * Used by the Compact Engine to determine which layer of detail
- * fits within the caller's token budget.
- */
-
-import { countTokens, isWithinTokenLimit } from 'gpt-tokenizer';
-
-/**
- * Count tokens in a string.
- */
-export function countTextTokens(text: string): number {
-  return countTokens(text);
-}
-
-/**
- * Check if text fits within a token limit.
- * Returns the token count if within limit, false otherwise.
- */
-export function fitsInBudget(text: string, limit: number): number | false {
-  return isWithinTokenLimit(text, limit);
-}
-
-/**
- * Truncate text to fit within a token budget.
- * Truncates at sentence boundaries when possible.
- */
-export function truncateToTokenBudget(text: string, budget: number): string {
-  if (fitsInBudget(text, budget) !== false) {
-    return text;
-  }
-
-  // Binary search for the right length
-  const sentences = text.split(/(?<=[.!?])\s+/);
-  let result = '';
-
-  for (const sentence of sentences) {
-    const candidate = result ? `${result} ${sentence}` : sentence;
-    if (fitsInBudget(candidate, budget) === false) {
-      break;
-    }
-    result = candidate;
-  }
-
-  // If no complete sentence fits, truncate by characters
-  if (!result) {
-    // Rough estimate: 1 token ≈ 4 chars for English, ≈ 1.5 chars for Chinese
-    const estimatedChars = budget * 2;
-    result = text.slice(0, estimatedChars);
-    // Refine
-    while (fitsInBudget(result, budget) === false && result.length > 0) {
-      result = result.slice(0, Math.floor(result.length * 0.9));
-    }
-    if (result.length < text.length) {
-      result += '...';
-    }
-  }
-
-  return result;
-}
-
-/**
- * Estimate the token cost of an IndexEntry line.
- * Used to predict compact index size.
- */
-export function estimateIndexEntryTokens(title: string): number {
-  // Format: "| #ID | Time | Icon | Title | ~Tokens |"
-  // Overhead is roughly 15 tokens for formatting
-  return countTextTokens(title) + 15;
-}
+/**
+ * Token Budget Manager
+ *
+ * Provides token counting and budget management for Progressive Disclosure.
+ * Source: gpt-tokenizer (737 stars, JS port of OpenAI's tiktoken)
+ *
+ * Used by the Compact Engine to determine which layer of detail
+ * fits within the caller's token budget.
+ */
+
+import { countTokens, isWithinTokenLimit } from 'gpt-tokenizer';
+
+/**
+ * Count tokens in a string.
+ */
+export function countTextTokens(text: string): number {
+  return countTokens(text);
+}
+
+/**
+ * Check if text fits within a token limit.
+ * Returns the token count if within limit, false otherwise.
+ */
+export function fitsInBudget(text: string, limit: number): number | false {
+  return isWithinTokenLimit(text, limit);
+}
+
+/**
+ * Truncate text to fit within a token budget.
+ * Truncates at sentence boundaries when possible.
+ */
+export function truncateToTokenBudget(text: string, budget: number): string {
+  if (fitsInBudget(text, budget) !== false) {
+    return text;
+  }
+
+  // Binary search for the right length
+  const sentences = text.split(/(?<=[.!?])\s+/);
+  let result = '';
+
+  for (const sentence of sentences) {
+    const candidate = result ? `${result} ${sentence}` : sentence;
+    if (fitsInBudget(candidate, budget) === false) {
+      break;
+    }
+    result = candidate;
+  }
+
+  // If no complete sentence fits, truncate by characters
+  if (!result) {
+    // Rough estimate: 1 token ≈ 4 chars for English, ≈ 1.5 chars for Chinese
+    const estimatedChars = budget * 2;
+    result = text.slice(0, estimatedChars);
+    // Refine
+    while (fitsInBudget(result, budget) === false && result.length > 0) {
+      result = result.slice(0, Math.floor(result.length * 0.9));
+    }
+    if (result.length < text.length) {
+      // A character estimate can end halfway through a flag, path, or symbol.
+      // Drop the incomplete whitespace-delimited token instead of returning
+      // misleading fragments such as `AUTH...`.
+      const nextCharacter = text.charAt(result.length);
+      if (nextCharacter && !/[\s.,;:!?)}\]]/.test(nextCharacter)) {
+        const boundary = result.search(/\s+\S*$/);
+        result = boundary > 0 ? result.slice(0, boundary).trimEnd() : '';
+      }
+
+      // Keep the suffix inside the stated budget when there is room. An empty
+      // prefix is more honest than a partial identifier that appears valid.
+      while (result && fitsInBudget(result + '...', budget) === false) {
+        const boundary = result.lastIndexOf(' ');
+        result = boundary > 0 ? result.slice(0, boundary).trimEnd() : '';
+      }
+      result = result ? result + '...' : '...';
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Estimate the token cost of an IndexEntry line.
+ * Used to predict compact index size.
+ */
+export function estimateIndexEntryTokens(title: string): number {
+  // Format: "| #ID | Time | Icon | Title | ~Tokens |"
+  // Overhead is roughly 15 tokens for formatting
+  return countTextTokens(title) + 15;
+}

package/src/config/toml-loader.ts

@@ -144,6 +144,9 @@ function parseTomlValue(raw: string, filePath: string, line: number): unknown {
   if (raw.startsWith('"') && raw.endsWith('"')) {
     return raw.slice(1, -1).replace(/\\"/g, '"').replace(/\\\\/g, '\\');
   }
+  if (raw.startsWith("'") && raw.endsWith("'")) {
+    return raw.slice(1, -1);
+  }
   if (raw === 'true') return true;
   if (raw === 'false') return false;
   if (/^-?\d+$/.test(raw)) return Number.parseInt(raw, 10);
@@ -157,7 +160,7 @@ function parseTomlValue(raw: string, filePath: string, line: number): unknown {
 }
 
 function stripComment(line: string): string {
-  let inString = false;
+  let quote: '"' | "'" | null = null;
   let escaped = false;
   for (let i = 0; i < line.length; i++) {
     const char = line[i];
@@ -165,15 +168,16 @@ function stripComment(line: string): string {
       escaped = false;
       continue;
     }
-    if (char === '\\' && inString) {
+    if (char === '\\' && quote === '"') {
       escaped = true;
       continue;
     }
-    if (char === '"') {
-      inString = !inString;
+    if (char === '"' || char === "'") {
+      if (quote === char) quote = null;
+      else if (quote === null) quote = char;
       continue;
     }
-    if (char === '#' && !inString) {
+    if (char === '#' && quote === null) {
       return line.slice(0, i);
     }
   }

package/src/dashboard/project-classification.ts

@@ -1,64 +1,64 @@
-/**
- * Project classification helpers — shared by /api/projects and /api/identity.
- *
- * Three kinds:
- *   - 'real':        genuine user projects (e.g. AVIDS2/memorix, github.com/org/repo)
- *   - 'temporary':   test/demo/smoke/e2e scratch projects (local/task-*, local/smoke-*, etc.)
- *   - 'placeholder': unresolved / obviously broken IDs (placeholder/*, __unresolved__, System32)
- *
- * A 'dirty' project is one with a clearly broken canonical ID (System32 etc.).
- * 'dirty' and 'temporary' are orthogonal axes:
- *   - local/task-abc        → temporary, NOT dirty
- *   - placeholder/xxx       → placeholder, dirty
- *   - System32\something    → real-looking location but dirty (broken ID)
- */
-
-export type ProjectKind = 'real' | 'temporary' | 'placeholder';
-
-/** Regex list — anything matching is temporary (scratch projects) */
-const TEMPORARY_PATTERNS: RegExp[] = [
-  /^local\/task-/i,
-  /^local\/smoke-/i,
-  /^local\/release-smoke-/i,
-  /^local\/memorix-e2e-/i,
-  /^local\/orchestrate-/i,
-  /^local\/scratch-/i,
-  /^local\/tmp-/i,
-];
-
-/** Regex list — anything matching is placeholder/unresolved */
-const PLACEHOLDER_PATTERNS: RegExp[] = [
-  /^__unresolved__$/,
-  /^placeholder\//i,
-];
-
-/** Regex list — IDs that indicate a broken canonical ID (dirty). */
-const DIRTY_PATTERNS: RegExp[] = [
-  /^placeholder\//i,
-  /System32/i,
-  /Microsoft VS Code/i,
-  /node_modules/i,
-  /\.vscode/i,
-  /^local\/[A-Z]:\\/i,
-];
-
-export function classifyProjectId(id: string): ProjectKind {
-  if (!id) return 'placeholder';
-  if (PLACEHOLDER_PATTERNS.some(p => p.test(id))) return 'placeholder';
-  if (TEMPORARY_PATTERNS.some(p => p.test(id))) return 'temporary';
-  return 'real';
-}
-
-export function isDirtyProjectId(id: string): boolean {
-  if (!id) return false;
-  return DIRTY_PATTERNS.some(p => p.test(id));
-}
-
-/** Friendly label for UI badges */
-export function projectKindLabel(kind: ProjectKind): string {
-  switch (kind) {
-    case 'real': return 'real';
-    case 'temporary': return 'temporary';
-    case 'placeholder': return 'placeholder';
-  }
-}
+/**
+ * Project classification helpers — shared by /api/projects and /api/identity.
+ *
+ * Three kinds:
+ *   - 'real':        genuine user projects (e.g. AVIDS2/memorix, github.com/org/repo)
+ *   - 'temporary':   test/demo/smoke/e2e scratch projects (local/task-*, local/smoke-*, etc.)
+ *   - 'placeholder': unresolved / obviously broken IDs (placeholder/*, __unresolved__, System32)
+ *
+ * A 'dirty' project is one with a clearly broken canonical ID (System32 etc.).
+ * 'dirty' and 'temporary' are orthogonal axes:
+ *   - local/task-abc        → temporary, NOT dirty
+ *   - placeholder/xxx       → placeholder, dirty
+ *   - System32\something    → real-looking location but dirty (broken ID)
+ */
+
+export type ProjectKind = 'real' | 'temporary' | 'placeholder';
+
+/** Regex list — anything matching is temporary (scratch projects) */
+const TEMPORARY_PATTERNS: RegExp[] = [
+  /^local\/task-/i,
+  /^local\/smoke-/i,
+  /^local\/release-smoke-/i,
+  /^local\/memorix-e2e-/i,
+  /^local\/orchestrate-/i,
+  /^local\/scratch-/i,
+  /^local\/tmp-/i,
+];
+
+/** Regex list — anything matching is placeholder/unresolved */
+const PLACEHOLDER_PATTERNS: RegExp[] = [
+  /^__unresolved__$/,
+  /^placeholder\//i,
+];
+
+/** Regex list — IDs that indicate a broken canonical ID (dirty). */
+const DIRTY_PATTERNS: RegExp[] = [
+  /^placeholder\//i,
+  /System32/i,
+  /Microsoft VS Code/i,
+  /node_modules/i,
+  /\.vscode/i,
+  /^local\/[A-Z]:\\/i,
+];
+
+export function classifyProjectId(id: string): ProjectKind {
+  if (!id) return 'placeholder';
+  if (PLACEHOLDER_PATTERNS.some(p => p.test(id))) return 'placeholder';
+  if (TEMPORARY_PATTERNS.some(p => p.test(id))) return 'temporary';
+  return 'real';
+}
+
+export function isDirtyProjectId(id: string): boolean {
+  if (!id) return false;
+  return DIRTY_PATTERNS.some(p => p.test(id));
+}
+
+/** Friendly label for UI badges */
+export function projectKindLabel(kind: ProjectKind): string {
+  switch (kind) {
+    case 'real': return 'real';
+    case 'temporary': return 'temporary';
+    case 'placeholder': return 'placeholder';
+  }
+}

package/src/embedding/fastembed-provider.ts

@@ -1,142 +1,142 @@
-/**
- * FastEmbed Provider
- *
- * Local ONNX-based embedding using fastembed (Qdrant).
- * Model: BAAI/bge-small-en-v1.5 (384 dimensions, ~30MB)
- *
- * This is an optional dependency — if fastembed is not installed,
- * the provider module gracefully falls back to fulltext-only search.
- *
- * Persistent disk cache: embeddings are saved to ~/.memorix/data/.embedding-cache.json
- * so server restarts don't need to regenerate them (saves minutes of CPU on 500+ obs).
- */
-
-import { createHash } from 'node:crypto';
-import { readFile, writeFile, mkdir } from 'node:fs/promises';
-import { join } from 'node:path';
-import { homedir } from 'node:os';
-import type { EmbeddingProvider } from './provider.js';
-
-const CACHE_DIR = process.env.MEMORIX_DATA_DIR || join(homedir(), '.memorix', 'data');
-const CACHE_FILE = join(CACHE_DIR, '.embedding-cache.json');
-
-// In-memory cache keyed by text hash → embedding
-const cache = new Map<string, number[]>();
-const MAX_CACHE_SIZE = 5000;
-let diskCacheDirty = false;
-
-function textHash(text: string): string {
-  return createHash('sha256').update(text).digest('hex').slice(0, 16);
-}
-
-async function loadDiskCache(): Promise<void> {
-  try {
-    const raw = await readFile(CACHE_FILE, 'utf-8');
-    const entries: [string, number[]][] = JSON.parse(raw);
-    for (const [k, v] of entries) cache.set(k, v);
-    console.error(`[memorix] Loaded ${entries.length} cached embeddings from disk`);
-  } catch {
-    // No cache file or corrupt — start fresh
-  }
-}
-
-async function saveDiskCache(): Promise<void> {
-  if (!diskCacheDirty) return;
-  try {
-    await mkdir(CACHE_DIR, { recursive: true });
-    const entries = Array.from(cache.entries());
-    await writeFile(CACHE_FILE, JSON.stringify(entries));
-    diskCacheDirty = false;
-  } catch {
-    // Ignore write errors — cache is an optimization, not critical
-  }
-}
-
-export class FastEmbedProvider implements EmbeddingProvider {
-  readonly name = 'fastembed-bge-small';
-  readonly dimensions = 384;
-
-  private model: { embed: (docs: string[], batchSize?: number) => AsyncGenerator<number[][]>; queryEmbed: (query: string) => Promise<number[]> };
-
-  private constructor(model: FastEmbedProvider['model']) {
-    this.model = model;
-  }
-
-  /**
-   * Initialize the FastEmbed provider.
-   * Downloads model on first use (~30MB), cached locally after.
-   * Loads persistent embedding cache from disk.
-   */
-  static async create(): Promise<FastEmbedProvider> {
-    // Dynamic import — throws if fastembed is not installed
-    const { EmbeddingModel, FlagEmbedding } = await import('fastembed');
-    const model = await FlagEmbedding.init({
-      model: EmbeddingModel.BGESmallENV15,
-    });
-    // Load disk cache before returning — subsequent embedBatch calls will hit cache
-    await loadDiskCache();
-    return new FastEmbedProvider(model);
-  }
-
-  async embed(text: string): Promise<number[]> {
-    const hash = textHash(text);
-    const cached = cache.get(hash);
-    if (cached) return cached;
-
-    const raw = await this.model.queryEmbed(text);
-    // Ensure plain number[] (fastembed may return Float32Array)
-    const result = Array.from(raw) as number[];
-    if (result.length !== this.dimensions) {
-      throw new Error(`Expected ${this.dimensions}d embedding, got ${result.length}d`);
-    }
-    this.cacheSet(hash, result);
-    return result;
-  }
-
-  async embedBatch(texts: string[]): Promise<number[][]> {
-    const results: number[][] = new Array(texts.length);
-    const uncachedIndices: number[] = [];
-    const uncachedTexts: string[] = [];
-
-    // Check cache for each text (by hash)
-    for (let i = 0; i < texts.length; i++) {
-      const hash = textHash(texts[i]);
-      const cached = cache.get(hash);
-      if (cached) {
-        results[i] = cached;
-      } else {
-        uncachedIndices.push(i);
-        uncachedTexts.push(texts[i]);
-      }
-    }
-
-    // Batch embed uncached texts
-    if (uncachedTexts.length > 0) {
-      console.error(`[memorix] Embedding ${uncachedTexts.length}/${texts.length} uncached texts (${texts.length - uncachedTexts.length} from cache)`);
-      let batchIdx = 0;
-      for await (const batch of this.model.embed(uncachedTexts, 64)) {
-        for (const vec of batch) {
-          const originalIdx = uncachedIndices[batchIdx];
-          const plain = Array.from(vec) as number[];
-          results[originalIdx] = plain;
-          this.cacheSet(textHash(uncachedTexts[batchIdx]), plain);
-          batchIdx++;
-        }
-      }
-      // Persist cache to disk after batch operations
-      await saveDiskCache();
-    }
-
-    return results;
-  }
-
-  private cacheSet(hash: string, value: number[]): void {
-    // Evict oldest entries if cache is full
-    if (cache.size >= MAX_CACHE_SIZE) {
-      const firstKey = cache.keys().next().value;
-      if (firstKey !== undefined) cache.delete(firstKey);
-    }
-    cache.set(hash, value);
-    diskCacheDirty = true;
-  }
-}
+/**
+ * FastEmbed Provider
+ *
+ * Local ONNX-based embedding using fastembed (Qdrant).
+ * Model: BAAI/bge-small-en-v1.5 (384 dimensions, ~30MB)
+ *
+ * This is an optional dependency — if fastembed is not installed,
+ * the provider module gracefully falls back to fulltext-only search.
+ *
+ * Persistent disk cache: embeddings are saved to ~/.memorix/data/.embedding-cache.json
+ * so server restarts don't need to regenerate them (saves minutes of CPU on 500+ obs).
+ */
+
+import { createHash } from 'node:crypto';
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+import type { EmbeddingProvider } from './provider.js';
+
+const CACHE_DIR = process.env.MEMORIX_DATA_DIR || join(homedir(), '.memorix', 'data');
+const CACHE_FILE = join(CACHE_DIR, '.embedding-cache.json');
+
+// In-memory cache keyed by text hash → embedding
+const cache = new Map<string, number[]>();
+const MAX_CACHE_SIZE = 5000;
+let diskCacheDirty = false;
+
+function textHash(text: string): string {
+  return createHash('sha256').update(text).digest('hex').slice(0, 16);
+}
+
+async function loadDiskCache(): Promise<void> {
+  try {
+    const raw = await readFile(CACHE_FILE, 'utf-8');
+    const entries: [string, number[]][] = JSON.parse(raw);
+    for (const [k, v] of entries) cache.set(k, v);
+    console.error(`[memorix] Loaded ${entries.length} cached embeddings from disk`);
+  } catch {
+    // No cache file or corrupt — start fresh
+  }
+}
+
+async function saveDiskCache(): Promise<void> {
+  if (!diskCacheDirty) return;
+  try {
+    await mkdir(CACHE_DIR, { recursive: true });
+    const entries = Array.from(cache.entries());
+    await writeFile(CACHE_FILE, JSON.stringify(entries));
+    diskCacheDirty = false;
+  } catch {
+    // Ignore write errors — cache is an optimization, not critical
+  }
+}
+
+export class FastEmbedProvider implements EmbeddingProvider {
+  readonly name = 'fastembed-bge-small';
+  readonly dimensions = 384;
+
+  private model: { embed: (docs: string[], batchSize?: number) => AsyncGenerator<number[][]>; queryEmbed: (query: string) => Promise<number[]> };
+
+  private constructor(model: FastEmbedProvider['model']) {
+    this.model = model;
+  }
+
+  /**
+   * Initialize the FastEmbed provider.
+   * Downloads model on first use (~30MB), cached locally after.
+   * Loads persistent embedding cache from disk.
+   */
+  static async create(): Promise<FastEmbedProvider> {
+    // Dynamic import — throws if fastembed is not installed
+    const { EmbeddingModel, FlagEmbedding } = await import('fastembed');
+    const model = await FlagEmbedding.init({
+      model: EmbeddingModel.BGESmallENV15,
+    });
+    // Load disk cache before returning — subsequent embedBatch calls will hit cache
+    await loadDiskCache();
+    return new FastEmbedProvider(model);
+  }
+
+  async embed(text: string): Promise<number[]> {
+    const hash = textHash(text);
+    const cached = cache.get(hash);
+    if (cached) return cached;
+
+    const raw = await this.model.queryEmbed(text);
+    // Ensure plain number[] (fastembed may return Float32Array)
+    const result = Array.from(raw) as number[];
+    if (result.length !== this.dimensions) {
+      throw new Error(`Expected ${this.dimensions}d embedding, got ${result.length}d`);
+    }
+    this.cacheSet(hash, result);
+    return result;
+  }
+
+  async embedBatch(texts: string[]): Promise<number[][]> {
+    const results: number[][] = new Array(texts.length);
+    const uncachedIndices: number[] = [];
+    const uncachedTexts: string[] = [];
+
+    // Check cache for each text (by hash)
+    for (let i = 0; i < texts.length; i++) {
+      const hash = textHash(texts[i]);
+      const cached = cache.get(hash);
+      if (cached) {
+        results[i] = cached;
+      } else {
+        uncachedIndices.push(i);
+        uncachedTexts.push(texts[i]);
+      }
+    }
+
+    // Batch embed uncached texts
+    if (uncachedTexts.length > 0) {
+      console.error(`[memorix] Embedding ${uncachedTexts.length}/${texts.length} uncached texts (${texts.length - uncachedTexts.length} from cache)`);
+      let batchIdx = 0;
+      for await (const batch of this.model.embed(uncachedTexts, 64)) {
+        for (const vec of batch) {
+          const originalIdx = uncachedIndices[batchIdx];
+          const plain = Array.from(vec) as number[];
+          results[originalIdx] = plain;
+          this.cacheSet(textHash(uncachedTexts[batchIdx]), plain);
+          batchIdx++;
+        }
+      }
+      // Persist cache to disk after batch operations
+      await saveDiskCache();
+    }
+
+    return results;
+  }
+
+  private cacheSet(hash: string, value: number[]): void {
+    // Evict oldest entries if cache is full
+    if (cache.size >= MAX_CACHE_SIZE) {
+      const firstKey = cache.keys().next().value;
+      if (firstKey !== undefined) cache.delete(firstKey);
+    }
+    cache.set(hash, value);
+    diskCacheDirty = true;
+  }
+}