clawmem 0.1.0

package/src/collections.ts

@@ -0,0 +1,405 @@
+/**
+ * Collections configuration management
+ *
+ * This module manages the YAML-based collection configuration at ~/.config/clawmem/config.yaml.
+ * Collections define which directories to index and their associated contexts.
+ */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+import YAML from "yaml";
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/**
+ * Context definitions for a collection
+ * Key is path prefix (e.g., "/", "/2024", "/Board of Directors")
+ * Value is the context description
+ */
+export type ContextMap = Record<string, string>;
+
+/**
+ * A single collection configuration
+ */
+export interface Collection {
+  path: string;           // Absolute path to index
+  pattern: string;        // Glob pattern (e.g., "**/*.md")
+  context?: ContextMap;   // Optional context definitions
+  update?: string;        // Optional bash command to run during qmd update
+}
+
+/**
+ * The complete configuration file structure
+ */
+export interface LifecyclePolicy {
+  archive_after_days: number;
+  type_overrides: Record<string, number | null>;
+  purge_after_days: number | null;
+  exempt_collections: string[];
+  dry_run: boolean;
+}
+
+export interface CollectionConfig {
+  global_context?: string;                    // Context applied to all collections
+  collections: Record<string, Collection>;    // Collection name -> config
+  directoryContext?: boolean;                 // Opt-in: auto-generate CLAUDE.md in directories
+  lifecycle?: LifecyclePolicy;                // Lifecycle management policy
+}
+
+/**
+ * Collection with its name (for return values)
+ */
+export interface NamedCollection extends Collection {
+  name: string;
+}
+
+// ============================================================================
+// Configuration paths
+// ============================================================================
+
+function getConfigDir(): string {
+  // Allow override via CLAWMEM_CONFIG_DIR for testing
+  if (process.env.CLAWMEM_CONFIG_DIR) {
+    return process.env.CLAWMEM_CONFIG_DIR;
+  }
+  return join(homedir(), ".config", "clawmem");
+}
+
+function getConfigFilePath(): string {
+  const dir = getConfigDir();
+  const preferred = join(dir, "config.yaml");
+  if (existsSync(preferred)) return preferred;
+  return join(dir, "index.yml");
+}
+
+/**
+ * Ensure config directory exists
+ */
+function ensureConfigDir(): void {
+  const configDir = getConfigDir();
+  if (!existsSync(configDir)) {
+    mkdirSync(configDir, { recursive: true });
+  }
+}
+
+// ============================================================================
+// Core functions
+// ============================================================================
+
+/**
+ * Load configuration from ~/.config/clawmem/config.yaml
+ * Returns empty config if file doesn't exist
+ */
+export function loadConfig(): CollectionConfig {
+  const configPath = getConfigFilePath();
+  if (!existsSync(configPath)) {
+    return { collections: {} };
+  }
+
+  try {
+    const content = readFileSync(configPath, "utf-8");
+    const config = YAML.parse(content) as CollectionConfig;
+
+    // Ensure collections object exists
+    if (!config.collections) {
+      config.collections = {};
+    }
+
+    // Parse lifecycle policy if present
+    const raw = YAML.parse(content);
+    if (raw?.lifecycle && typeof raw.lifecycle === "object") {
+      const lc = raw.lifecycle;
+      config.lifecycle = {
+        archive_after_days: typeof lc.archive_after_days === "number" ? lc.archive_after_days : 90,
+        type_overrides: typeof lc.type_overrides === "object" && lc.type_overrides !== null ? lc.type_overrides : {},
+        purge_after_days: typeof lc.purge_after_days === "number" ? lc.purge_after_days : null,
+        exempt_collections: Array.isArray(lc.exempt_collections) ? lc.exempt_collections : [],
+        dry_run: lc.dry_run !== false,
+      };
+    }
+
+    return config;
+  } catch (error) {
+    throw new Error(`Failed to parse ${configPath}: ${error}`);
+  }
+}
+
+/**
+ * Save configuration to ~/.config/clawmem/index.yml
+ */
+export function saveConfig(config: CollectionConfig): void {
+  ensureConfigDir();
+  const configPath = getConfigFilePath();
+
+  try {
+    const yaml = YAML.stringify(config, {
+      indent: 2,
+      lineWidth: 0,  // Don't wrap lines
+    });
+    writeFileSync(configPath, yaml, "utf-8");
+  } catch (error) {
+    throw new Error(`Failed to write ${configPath}: ${error}`);
+  }
+}
+
+/**
+ * Get a specific collection by name
+ * Returns null if not found
+ */
+export function getCollection(name: string): NamedCollection | null {
+  const config = loadConfig();
+  const collection = config.collections[name];
+
+  if (!collection) {
+    return null;
+  }
+
+  return { name, ...collection };
+}
+
+/**
+ * List all collections
+ */
+export function listCollections(): NamedCollection[] {
+  const config = loadConfig();
+  return Object.entries(config.collections).map(([name, collection]) => ({
+    name,
+    ...collection,
+  }));
+}
+
+/**
+ * Add or update a collection
+ */
+export function addCollection(
+  name: string,
+  path: string,
+  pattern: string = "**/*.md"
+): void {
+  const config = loadConfig();
+
+  config.collections[name] = {
+    path,
+    pattern,
+    context: config.collections[name]?.context, // Preserve existing context
+  };
+
+  saveConfig(config);
+}
+
+/**
+ * Remove a collection
+ */
+export function removeCollection(name: string): boolean {
+  const config = loadConfig();
+
+  if (!config.collections[name]) {
+    return false;
+  }
+
+  delete config.collections[name];
+  saveConfig(config);
+  return true;
+}
+
+/**
+ * Rename a collection
+ */
+export function renameCollection(oldName: string, newName: string): boolean {
+  const config = loadConfig();
+
+  if (!config.collections[oldName]) {
+    return false;
+  }
+
+  if (config.collections[newName]) {
+    throw new Error(`Collection '${newName}' already exists`);
+  }
+
+  config.collections[newName] = config.collections[oldName];
+  delete config.collections[oldName];
+  saveConfig(config);
+  return true;
+}
+
+// ============================================================================
+// Context management
+// ============================================================================
+
+/**
+ * Get global context
+ */
+export function getGlobalContext(): string | undefined {
+  const config = loadConfig();
+  return config.global_context;
+}
+
+/**
+ * Set global context
+ */
+export function setGlobalContext(context: string | undefined): void {
+  const config = loadConfig();
+  config.global_context = context;
+  saveConfig(config);
+}
+
+/**
+ * Get all contexts for a collection
+ */
+export function getContexts(collectionName: string): ContextMap | undefined {
+  const collection = getCollection(collectionName);
+  return collection?.context;
+}
+
+/**
+ * Add or update a context for a specific path in a collection
+ */
+export function addContext(
+  collectionName: string,
+  pathPrefix: string,
+  contextText: string
+): boolean {
+  const config = loadConfig();
+  const collection = config.collections[collectionName];
+
+  if (!collection) {
+    return false;
+  }
+
+  if (!collection.context) {
+    collection.context = {};
+  }
+
+  collection.context[pathPrefix] = contextText;
+  saveConfig(config);
+  return true;
+}
+
+/**
+ * Remove a context from a collection
+ */
+export function removeContext(
+  collectionName: string,
+  pathPrefix: string
+): boolean {
+  const config = loadConfig();
+  const collection = config.collections[collectionName];
+
+  if (!collection?.context?.[pathPrefix]) {
+    return false;
+  }
+
+  delete collection.context[pathPrefix];
+
+  // Remove empty context object
+  if (Object.keys(collection.context).length === 0) {
+    delete collection.context;
+  }
+
+  saveConfig(config);
+  return true;
+}
+
+/**
+ * List all contexts across all collections
+ */
+export function listAllContexts(): Array<{
+  collection: string;
+  path: string;
+  context: string;
+}> {
+  const config = loadConfig();
+  const results: Array<{ collection: string; path: string; context: string }> = [];
+
+  // Add global context if present
+  if (config.global_context) {
+    results.push({
+      collection: "*",
+      path: "/",
+      context: config.global_context,
+    });
+  }
+
+  // Add collection contexts
+  for (const [name, collection] of Object.entries(config.collections)) {
+    if (collection.context) {
+      for (const [path, context] of Object.entries(collection.context)) {
+        results.push({
+          collection: name,
+          path,
+          context,
+        });
+      }
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Find best matching context for a given collection and path
+ * Returns the most specific matching context (longest path prefix match)
+ */
+export function findContextForPath(
+  collectionName: string,
+  filePath: string
+): string | undefined {
+  const config = loadConfig();
+  const collection = config.collections[collectionName];
+
+  if (!collection?.context) {
+    return config.global_context;
+  }
+
+  // Find all matching prefixes
+  const matches: Array<{ prefix: string; context: string }> = [];
+
+  for (const [prefix, context] of Object.entries(collection.context)) {
+    // Normalize paths for comparison
+    const normalizedPath = filePath.startsWith("/") ? filePath : `/${filePath}`;
+    const normalizedPrefix = prefix.startsWith("/") ? prefix : `/${prefix}`;
+
+    if (normalizedPath.startsWith(normalizedPrefix)) {
+      matches.push({ prefix: normalizedPrefix, context });
+    }
+  }
+
+  // Return most specific match (longest prefix)
+  if (matches.length > 0) {
+    matches.sort((a, b) => b.prefix.length - a.prefix.length);
+    return matches[0]!.context;
+  }
+
+  // Fallback to global context
+  return config.global_context;
+}
+
+// ============================================================================
+// Utility functions
+// ============================================================================
+
+/**
+ * Get the config file path (useful for error messages)
+ */
+export function getConfigPath(): string {
+  return getConfigFilePath();
+}
+
+/**
+ * Check if config file exists
+ */
+export function configExists(): boolean {
+  return existsSync(getConfigFilePath());
+}
+
+/**
+ * Validate a collection name
+ * Collection names must be valid and not contain special characters
+ */
+export function isValidCollectionName(name: string): boolean {
+  // Allow alphanumeric, hyphens, underscores
+  return /^[a-zA-Z0-9_-]+$/.test(name);
+}

package/src/config.ts

@@ -0,0 +1,178 @@
+/**
+ * ClawMem Configuration — Vault routing, lifecycle policy, performance profiles.
+ *
+ * Multi-vault support: ClawMem can manage multiple independent SQLite vaults,
+ * each with its own documents, embeddings, and graphs. The default (unnamed)
+ * vault lives at ~/.cache/clawmem/index.sqlite. Named vaults are configured
+ * via config.yaml or environment variables.
+ *
+ * Single vault is the default. Multi-vault is opt-in.
+ *
+ * Configuration sources (highest priority first):
+ *   1. Environment variables (CLAWMEM_VAULTS JSON map)
+ *   2. Config file (~/.config/clawmem/config.yaml, vaults section)
+ *
+ * Example config.yaml with multiple vaults:
+ *   vaults:
+ *     work: ~/.cache/clawmem/work.sqlite
+ *     personal: ~/.cache/clawmem/personal.sqlite
+ *
+ * When no vaults are configured, ClawMem operates as a single-vault system.
+ * All tools work without the vault parameter — it's always optional.
+ */
+
+import { existsSync, readFileSync } from "fs";
+import { resolve, join } from "path";
+import { homedir } from "os";
+import YAML from "yaml";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/** Expand leading ~ to user home directory */
+function expandHome(p: string): string {
+  return p.startsWith("~/") || p === "~" ? join(homedir(), p.slice(1)) : p;
+}
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface VaultConfig {
+  /** Vault name → absolute path to SQLite file */
+  [name: string]: string;
+}
+
+export interface LifecyclePolicy {
+  archive_after_days: number;
+  type_overrides: Record<string, number | null>;
+  purge_after_days: number | null;
+  exempt_collections: string[];
+  dry_run: boolean;
+}
+
+export interface ClawMemConfig {
+  /** Named vault paths (empty = single-vault mode) */
+  vaults: VaultConfig;
+  /** Lifecycle management policy */
+  lifecycle?: LifecyclePolicy;
+}
+
+// ---------------------------------------------------------------------------
+// Performance Profiles
+// ---------------------------------------------------------------------------
+
+export type PerformanceProfile = "speed" | "balanced" | "deep";
+
+export interface ProfileConfig {
+  tokenBudget: number;
+  maxResults: number;
+  useVector: boolean;
+  vectorTimeout: number;
+  minScore: number;
+}
+
+export const PROFILES: Record<PerformanceProfile, ProfileConfig> = {
+  speed:    { tokenBudget: 400,  maxResults: 5,  useVector: false, vectorTimeout: 0,    minScore: 0.55 },
+  balanced: { tokenBudget: 800,  maxResults: 10, useVector: true,  vectorTimeout: 900,  minScore: 0.45 },
+  deep:     { tokenBudget: 1200, maxResults: 15, useVector: true,  vectorTimeout: 2000, minScore: 0.35 },
+};
+
+export function getActiveProfile(): ProfileConfig {
+  const profileName = (process.env.CLAWMEM_PROFILE || "balanced") as PerformanceProfile;
+  return PROFILES[profileName] || PROFILES.balanced;
+}
+
+// ---------------------------------------------------------------------------
+// Config loading
+// ---------------------------------------------------------------------------
+
+let _cachedConfig: ClawMemConfig | null = null;
+
+/**
+ * Load vault configuration from env vars and config file.
+ * Priority: env vars override config file values.
+ */
+export function loadVaultConfig(): ClawMemConfig {
+  if (_cachedConfig) return _cachedConfig;
+
+  const vaults: VaultConfig = {};
+
+  // 1. Load from config.yaml (vaults section)
+  const configDir = process.env.CLAWMEM_CONFIG_DIR || join(homedir(), ".config", "clawmem");
+  const configPath = join(configDir, "config.yaml");
+
+  let parsedYaml: any = null;
+  if (existsSync(configPath)) {
+    try {
+      const content = readFileSync(configPath, "utf-8");
+      parsedYaml = YAML.parse(content);
+      if (parsedYaml?.vaults && typeof parsedYaml.vaults === "object") {
+        for (const [name, path] of Object.entries(parsedYaml.vaults)) {
+          if (typeof path === "string") {
+            vaults[name] = resolve(expandHome(path));
+          }
+        }
+      }
+    } catch {
+      // Config parse failure — continue with env vars only
+    }
+  }
+
+  // 2. Override with env vars (higher priority)
+  if (process.env.CLAWMEM_VAULTS) {
+    try {
+      const envVaults = JSON.parse(process.env.CLAWMEM_VAULTS);
+      if (typeof envVaults === "object") {
+        for (const [name, path] of Object.entries(envVaults)) {
+          if (typeof path === "string") {
+            vaults[name] = resolve(expandHome(path as string));
+          }
+        }
+      }
+    } catch {
+      // Invalid JSON — ignore
+    }
+  }
+
+  // 3. Lifecycle policy (optional)
+  let lifecycle: LifecyclePolicy | undefined;
+  if (parsedYaml?.lifecycle && typeof parsedYaml.lifecycle === "object") {
+    const lc = parsedYaml.lifecycle;
+    lifecycle = {
+      archive_after_days: typeof lc.archive_after_days === "number" ? lc.archive_after_days : 90,
+      type_overrides: typeof lc.type_overrides === "object" && lc.type_overrides !== null ? lc.type_overrides : {},
+      purge_after_days: typeof lc.purge_after_days === "number" ? lc.purge_after_days : null,
+      exempt_collections: Array.isArray(lc.exempt_collections) ? lc.exempt_collections : [],
+      dry_run: lc.dry_run !== false,
+    };
+  }
+
+  _cachedConfig = { vaults, lifecycle };
+  return _cachedConfig;
+}
+
+/**
+ * Get the SQLite path for a named vault.
+ * Returns undefined if vault is not configured.
+ */
+export function getVaultPath(vaultName: string): string | undefined {
+  const config = loadVaultConfig();
+  return config.vaults[vaultName];
+}
+
+/**
+ * List all configured vault names.
+ */
+export function listVaults(): string[] {
+  const config = loadVaultConfig();
+  return Object.keys(config.vaults);
+}
+
+/**
+ * Clear cached config (for testing or after env var changes).
+ */
+export function clearConfigCache(): void {
+  _cachedConfig = null;
+}

package/src/consolidation.ts

@@ -0,0 +1,123 @@
+/**
+ * ClawMem Consolidation Worker
+ *
+ * Background worker that enriches documents missing A-MEM metadata.
+ * Runs periodically to backfill memory notes for documents indexed before A-MEM.
+ */
+
+import type { Store } from "./store.ts";
+import type { LlamaCpp } from "./llm.ts";
+
+// =============================================================================
+// Types
+// =============================================================================
+
+interface DocumentToEnrich {
+  id: number;
+  hash: string;
+  title: string;
+}
+
+// =============================================================================
+// Worker State
+// =============================================================================
+
+let consolidationTimer: Timer | null = null;
+let isRunning = false;
+
+// =============================================================================
+// Worker Functions
+// =============================================================================
+
+/**
+ * Starts the consolidation worker that enriches documents missing A-MEM metadata.
+ *
+ * @param store - Store instance with A-MEM methods
+ * @param llm - LLM instance for memory note construction
+ * @param intervalMs - Tick interval in milliseconds (default: 300000 = 5 min)
+ */
+export function startConsolidationWorker(
+  store: Store,
+  llm: LlamaCpp,
+  intervalMs: number = 300000
+): void {
+  // Clamp interval to minimum 15 seconds
+  const interval = Math.max(15000, intervalMs);
+
+  console.log(`[consolidation] Starting worker with ${interval}ms interval`);
+
+  // Set up periodic tick
+  consolidationTimer = setInterval(async () => {
+    await tick(store, llm);
+  }, interval);
+
+  // Use unref() to avoid blocking process exit
+  consolidationTimer.unref();
+
+  console.log("[consolidation] Worker started");
+}
+
+/**
+ * Stops the consolidation worker.
+ */
+export function stopConsolidationWorker(): void {
+  if (consolidationTimer) {
+    clearInterval(consolidationTimer);
+    consolidationTimer = null;
+    console.log("[consolidation] Worker stopped");
+  }
+}
+
+/**
+ * Single worker tick: find and enrich up to 3 documents missing A-MEM metadata.
+ */
+async function tick(store: Store, llm: LlamaCpp): Promise<void> {
+  // Reentrancy guard
+  if (isRunning) {
+    console.log("[consolidation] Skipping tick (already running)");
+    return;
+  }
+
+  isRunning = true;
+
+  try {
+    // Find documents missing A-MEM keywords (primary indicator of unenriched docs)
+    const docs = store.db
+      .prepare<DocumentToEnrich, []>(
+        `SELECT id, hash, title
+         FROM documents
+         WHERE amem_keywords IS NULL AND active = 1
+         ORDER BY created_at ASC
+         LIMIT 3`
+      )
+      .all();
+
+    if (docs.length === 0) {
+      // No work to do
+      return;
+    }
+
+    console.log(`[consolidation] Enriching ${docs.length} documents`);
+
+    // Enrich each document (note + links, skip evolution to avoid cascades)
+    for (const doc of docs) {
+      try {
+        // Construct and store memory note
+        const note = await store.constructMemoryNote(llm, doc.id);
+        await store.storeMemoryNote(doc.id, note);
+
+        // Generate memory links (skip evolution for backlog)
+        await store.generateMemoryLinks(llm, doc.id);
+
+        console.log(`[consolidation] Enriched doc ${doc.id} (${doc.title})`);
+      } catch (err) {
+        console.error(`[consolidation] Failed to enrich doc ${doc.id}:`, err);
+        // Continue with remaining docs (don't let one failure block the queue)
+      }
+    }
+  } catch (err) {
+    console.error("[consolidation] Tick failed:", err);
+  } finally {
+    isRunning = false;
+  }
+}