@soleri/core 9.0.4 → 9.3.0

package/src/vault/vault-markdown-sync.ts

@@ -0,0 +1,163 @@
+/**
+ * Vault Markdown Sync — auto-sync vault entries to browsable markdown files.
+ *
+ * Writes entries as markdown with YAML frontmatter to knowledge/vault/{domain}/{slug}.md
+ * for offline browsability. Reuses patterns from obsidian-sync.ts.
+ */
+
+import { mkdirSync, writeFileSync, readFileSync, readdirSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import type { IntelligenceEntry } from '../intelligence/types.js';
+import { computeContentHash } from './content-hash.js';
+import type { Vault } from './vault.js';
+
+// ─── Format ─────────────────────────────────────────────────────────
+
+/** Slugify a title for use as a filename. */
+export function titleToSlug(title: string): string {
+  return title
+    .toLowerCase()
+    .replace(/[^\w\s-]/g, '')
+    .replace(/\s+/g, '-')
+    .replace(/-+/g, '-')
+    .replace(/^-|-$/g, '')
+    .slice(0, 80);
+}
+
+/** Convert a vault entry to markdown string with YAML frontmatter. */
+export function entryToMarkdown(entry: IntelligenceEntry): string {
+  const lines: string[] = ['---'];
+  lines.push(`id: "${entry.id}"`);
+  lines.push(`type: "${entry.type}"`);
+  if (entry.domain) lines.push(`domain: "${entry.domain}"`);
+  if (entry.tags && entry.tags.length > 0) {
+    lines.push(`tags: [${entry.tags.map((t) => `"${t}"`).join(', ')}]`);
+  }
+  if (entry.severity) lines.push(`severity: "${entry.severity}"`);
+  if (entry.tier) lines.push(`tier: "${entry.tier}"`);
+  if (entry.origin) lines.push(`origin: "${entry.origin}"`);
+  const created = entry.validFrom
+    ? new Date(entry.validFrom * 1000).toISOString().slice(0, 10)
+    : new Date().toISOString().slice(0, 10);
+  lines.push(`created: ${created}`);
+  lines.push(`content_hash: "${computeContentHash(entry)}"`);
+  lines.push('---');
+  lines.push('');
+  lines.push(`# ${entry.title}`);
+  lines.push('');
+  lines.push(entry.description);
+  if (entry.context) {
+    lines.push('');
+    lines.push('## Context');
+    lines.push('');
+    lines.push(entry.context);
+  }
+  if (entry.example) {
+    lines.push('');
+    lines.push('## Example');
+    lines.push('');
+    lines.push(entry.example);
+  }
+  if (entry.counterExample) {
+    lines.push('');
+    lines.push('## Counter-Example');
+    lines.push('');
+    lines.push(entry.counterExample);
+  }
+  if (entry.why) {
+    lines.push('');
+    lines.push('## Why');
+    lines.push('');
+    lines.push(entry.why);
+  }
+  lines.push('');
+  return lines.join('\n');
+}
+
+// ─── Sync ───────────────────────────────────────────────────────────
+
+/** Write a single entry as a markdown file to knowledge/vault/{domain}/{slug}.md */
+export async function syncEntryToMarkdown(
+  entry: IntelligenceEntry,
+  knowledgeDir: string,
+): Promise<void> {
+  const domain = entry.domain || '_general';
+  const slug = titleToSlug(entry.title);
+  if (!slug) return;
+
+  const dir = join(knowledgeDir, 'vault', domain);
+  mkdirSync(dir, { recursive: true });
+
+  const filePath = join(dir, `${slug}.md`);
+  const content = entryToMarkdown(entry);
+  writeFileSync(filePath, content, 'utf-8');
+}
+
+/** Sync all vault entries to markdown, skipping entries whose content hash matches. */
+export async function syncAllToMarkdown(
+  vault: Vault,
+  knowledgeDir: string,
+): Promise<{ synced: number; skipped: number }> {
+  const entries = vault.list({ limit: 10000 });
+  let synced = 0;
+  let skipped = 0;
+
+  for (const entry of entries) {
+    const domain = entry.domain || '_general';
+    const slug = titleToSlug(entry.title);
+    if (!slug) {
+      skipped++;
+      continue;
+    }
+
+    const filePath = join(knowledgeDir, 'vault', domain, `${slug}.md`);
+    if (existsSync(filePath)) {
+      const existing = readFileSync(filePath, 'utf-8');
+      const hashMatch = existing.match(/^content_hash:\s*"([^"]+)"/m);
+      if (hashMatch && hashMatch[1] === computeContentHash(entry)) {
+        skipped++;
+        continue;
+      }
+    }
+
+    await syncEntryToMarkdown(entry, knowledgeDir);
+    synced++;
+  }
+
+  await generateIndex(knowledgeDir);
+  return { synced, skipped };
+}
+
+// ─── Index ──────────────────────────────────────────────────────────
+
+/** Generate _index.md with entry counts per domain. */
+export async function generateIndex(knowledgeDir: string): Promise<void> {
+  const vaultDir = join(knowledgeDir, 'vault');
+  if (!existsSync(vaultDir)) return;
+
+  const domains: Array<{ name: string; count: number }> = [];
+  const entries = readdirSync(vaultDir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    const domainDir = join(vaultDir, entry.name);
+    const files = readdirSync(domainDir).filter((f) => f.endsWith('.md'));
+    domains.push({ name: entry.name, count: files.length });
+  }
+
+  domains.sort((a, b) => b.count - a.count);
+  const total = domains.reduce((sum, d) => sum + d.count, 0);
+
+  const lines: string[] = [
+    '# Vault Knowledge Index',
+    '',
+    `> Auto-generated. ${total} entries across ${domains.length} domains.`,
+    '',
+    '| Domain | Entries |',
+    '|--------|---------|',
+    ...domains.map((d) => `| [${d.name}](./${d.name}/) | ${d.count} |`),
+    '',
+  ];
+
+  writeFileSync(join(vaultDir, '_index.md'), lines.join('\n'), 'utf-8');
+}

package/src/vault/vault-memories.ts

@@ -0,0 +1,339 @@
+/**
+ * Vault memory/session operations — capture, search, list, export, import, prune, dedup.
+ * Extracted from vault.ts as part of Wave 0C decomposition.
+ */
+import type { PersistenceProvider } from '../persistence/types.js';
+import type { Memory, MemoryStats } from './vault.js';
+
+export function captureMemory(
+  provider: PersistenceProvider,
+  memory: Omit<Memory, 'id' | 'createdAt' | 'archivedAt'>,
+): Memory {
+  const id = `mem-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+  provider.run(
+    `INSERT INTO memories (id, project_path, type, context, summary, topics, files_modified, tools_used, intent, decisions, current_state, next_steps, vault_entries_referenced)
+     VALUES (@id, @projectPath, @type, @context, @summary, @topics, @filesModified, @toolsUsed, @intent, @decisions, @currentState, @nextSteps, @vaultEntriesReferenced)`,
+    {
+      id,
+      projectPath: memory.projectPath,
+      type: memory.type,
+      context: memory.context,
+      summary: memory.summary,
+      topics: JSON.stringify(memory.topics),
+      filesModified: JSON.stringify(memory.filesModified),
+      toolsUsed: JSON.stringify(memory.toolsUsed),
+      intent: memory.intent ?? null,
+      decisions: JSON.stringify(memory.decisions ?? []),
+      currentState: memory.currentState ?? null,
+      nextSteps: JSON.stringify(memory.nextSteps ?? []),
+      vaultEntriesReferenced: JSON.stringify(memory.vaultEntriesReferenced ?? []),
+    },
+  );
+  return getMemory(provider, id)!;
+}
+
+export function getMemory(provider: PersistenceProvider, id: string): Memory | null {
+  const row = provider.get<Record<string, unknown>>('SELECT * FROM memories WHERE id = ?', [id]);
+  return row ? rowToMemory(row) : null;
+}
+
+export function deleteMemory(provider: PersistenceProvider, id: string): boolean {
+  return provider.run('DELETE FROM memories WHERE id = ?', [id]).changes > 0;
+}
+
+export function searchMemories(
+  provider: PersistenceProvider,
+  query: string,
+  options?: { type?: string; projectPath?: string; intent?: string; limit?: number },
+): Memory[] {
+  const limit = options?.limit ?? 10;
+  const filters: string[] = ['m.archived_at IS NULL'];
+  const fp: Record<string, unknown> = {};
+  if (options?.type) {
+    filters.push('m.type = @type');
+    fp.type = options.type;
+  }
+  if (options?.projectPath) {
+    filters.push('m.project_path = @projectPath');
+    fp.projectPath = options.projectPath;
+  }
+  if (options?.intent) {
+    filters.push('m.intent = @intent');
+    fp.intent = options.intent;
+  }
+  const wc = filters.length > 0 ? `AND ${filters.join(' AND ')}` : '';
+  try {
+    const rows = provider.all<Record<string, unknown>>(
+      `SELECT m.* FROM memories_fts fts JOIN memories m ON m.rowid = fts.rowid WHERE memories_fts MATCH @query ${wc} ORDER BY rank LIMIT @limit`,
+      { query, limit, ...fp },
+    );
+    return rows.map(rowToMemory);
+  } catch {
+    return [];
+  }
+}
+
+export function listMemories(
+  provider: PersistenceProvider,
+  options?: { type?: string; projectPath?: string; limit?: number; offset?: number },
+): Memory[] {
+  const filters: string[] = ['archived_at IS NULL'];
+  const params: Record<string, unknown> = {};
+  if (options?.type) {
+    filters.push('type = @type');
+    params.type = options.type;
+  }
+  if (options?.projectPath) {
+    filters.push('project_path = @projectPath');
+    params.projectPath = options.projectPath;
+  }
+  const wc = `WHERE ${filters.join(' AND ')}`;
+  const rows = provider.all<Record<string, unknown>>(
+    `SELECT * FROM memories ${wc} ORDER BY created_at DESC LIMIT @limit OFFSET @offset`,
+    { ...params, limit: options?.limit ?? 50, offset: options?.offset ?? 0 },
+  );
+  return rows.map(rowToMemory);
+}
+
+export function memoryStats(provider: PersistenceProvider): MemoryStats {
+  const total = provider.get<{ count: number }>(
+    'SELECT COUNT(*) as count FROM memories WHERE archived_at IS NULL',
+  )!.count;
+  const byTypeRows = provider.all<{ key: string; count: number }>(
+    'SELECT type as key, COUNT(*) as count FROM memories WHERE archived_at IS NULL GROUP BY type',
+  );
+  const byProjectRows = provider.all<{ key: string; count: number }>(
+    'SELECT project_path as key, COUNT(*) as count FROM memories WHERE archived_at IS NULL GROUP BY project_path',
+  );
+  return {
+    total,
+    byType: Object.fromEntries(byTypeRows.map((r) => [r.key, r.count])),
+    byProject: Object.fromEntries(byProjectRows.map((r) => [r.key, r.count])),
+  };
+}
+
+export function memoryStatsDetailed(
+  provider: PersistenceProvider,
+  options?: { projectPath?: string; fromDate?: number; toDate?: number },
+): MemoryStats & { oldest: number | null; newest: number | null; archivedCount: number } {
+  const filters: string[] = [];
+  const params: Record<string, unknown> = {};
+  if (options?.projectPath) {
+    filters.push('project_path = @projectPath');
+    params.projectPath = options.projectPath;
+  }
+  if (options?.fromDate) {
+    filters.push('created_at >= @fromDate');
+    params.fromDate = options.fromDate;
+  }
+  if (options?.toDate) {
+    filters.push('created_at <= @toDate');
+    params.toDate = options.toDate;
+  }
+  const wc = filters.length > 0 ? `WHERE ${filters.join(' AND ')}` : '';
+
+  const total = provider.get<{ count: number }>(
+    `SELECT COUNT(*) as count FROM memories ${wc}${wc ? ' AND' : ' WHERE'} archived_at IS NULL`,
+    params,
+  )!.count;
+
+  const archivedCount = provider.get<{ count: number }>(
+    `SELECT COUNT(*) as count FROM memories ${wc}${wc ? ' AND' : ' WHERE'} archived_at IS NOT NULL`,
+    params,
+  )!.count;
+
+  const byTypeRows = provider.all<{ key: string; count: number }>(
+    `SELECT type as key, COUNT(*) as count FROM memories ${wc}${wc ? ' AND' : ' WHERE'} archived_at IS NULL GROUP BY type`,
+    params,
+  );
+
+  const byProjectRows = provider.all<{ key: string; count: number }>(
+    `SELECT project_path as key, COUNT(*) as count FROM memories ${wc}${wc ? ' AND' : ' WHERE'} archived_at IS NULL GROUP BY project_path`,
+    params,
+  );
+
+  const dateRange = provider.get<{ oldest: number | null; newest: number | null }>(
+    `SELECT MIN(created_at) as oldest, MAX(created_at) as newest FROM memories ${wc}${wc ? ' AND' : ' WHERE'} archived_at IS NULL`,
+    params,
+  )!;
+
+  return {
+    total,
+    byType: Object.fromEntries(byTypeRows.map((r) => [r.key, r.count])),
+    byProject: Object.fromEntries(byProjectRows.map((r) => [r.key, r.count])),
+    oldest: dateRange.oldest,
+    newest: dateRange.newest,
+    archivedCount,
+  };
+}
+
+export function exportMemories(
+  provider: PersistenceProvider,
+  options?: { projectPath?: string; type?: string; includeArchived?: boolean },
+): Memory[] {
+  const filters: string[] = [];
+  const params: Record<string, unknown> = {};
+  if (!options?.includeArchived) {
+    filters.push('archived_at IS NULL');
+  }
+  if (options?.projectPath) {
+    filters.push('project_path = @projectPath');
+    params.projectPath = options.projectPath;
+  }
+  if (options?.type) {
+    filters.push('type = @type');
+    params.type = options.type;
+  }
+  const wc = filters.length > 0 ? `WHERE ${filters.join(' AND ')}` : '';
+  const rows = provider.all<Record<string, unknown>>(
+    `SELECT * FROM memories ${wc} ORDER BY created_at ASC`,
+    Object.keys(params).length > 0 ? params : undefined,
+  );
+  return rows.map(rowToMemory);
+}
+
+export function importMemories(
+  provider: PersistenceProvider,
+  memories: Memory[],
+): { imported: number; skipped: number } {
+  const sql = `
+    INSERT OR IGNORE INTO memories (id, project_path, type, context, summary, topics, files_modified, tools_used, created_at, archived_at)
+    VALUES (@id, @projectPath, @type, @context, @summary, @topics, @filesModified, @toolsUsed, @createdAt, @archivedAt)
+  `;
+  let imported = 0;
+  let skipped = 0;
+  provider.transaction(() => {
+    for (const m of memories) {
+      const result = provider.run(sql, {
+        id: m.id,
+        projectPath: m.projectPath,
+        type: m.type,
+        context: m.context,
+        summary: m.summary,
+        topics: JSON.stringify(m.topics),
+        filesModified: JSON.stringify(m.filesModified),
+        toolsUsed: JSON.stringify(m.toolsUsed),
+        createdAt: m.createdAt,
+        archivedAt: m.archivedAt,
+      });
+      if (result.changes > 0) imported++;
+      else skipped++;
+    }
+  });
+  return { imported, skipped };
+}
+
+export function pruneMemories(
+  provider: PersistenceProvider,
+  olderThanDays: number,
+): { pruned: number } {
+  const cutoff = Math.floor(Date.now() / 1000) - olderThanDays * 86400;
+  const result = provider.run('DELETE FROM memories WHERE created_at < ? AND archived_at IS NULL', [
+    cutoff,
+  ]);
+  return { pruned: result.changes };
+}
+
+export function deduplicateMemories(provider: PersistenceProvider): {
+  removed: number;
+  groups: Array<{ kept: string; removed: string[] }>;
+} {
+  const dupeRows = provider.all<{ id1: string; id2: string }>(`
+      SELECT m1.id as id1, m2.id as id2
+      FROM memories m1
+      JOIN memories m2 ON m1.summary = m2.summary
+        AND m1.project_path = m2.project_path
+        AND m1.type = m2.type
+        AND m1.id < m2.id
+        AND m1.archived_at IS NULL
+        AND m2.archived_at IS NULL
+    `);
+
+  const groupMap = new Map<string, Set<string>>();
+  for (const row of dupeRows) {
+    if (!groupMap.has(row.id1)) groupMap.set(row.id1, new Set());
+    groupMap.get(row.id1)!.add(row.id2);
+  }
+
+  const groups: Array<{ kept: string; removed: string[] }> = [];
+  const toRemove = new Set<string>();
+  for (const [kept, removedSet] of groupMap) {
+    const removed = [...removedSet].filter((id) => !toRemove.has(id));
+    if (removed.length > 0) {
+      groups.push({ kept, removed });
+      for (const id of removed) toRemove.add(id);
+    }
+  }
+
+  if (toRemove.size > 0) {
+    provider.transaction(() => {
+      for (const id of toRemove) {
+        provider.run('DELETE FROM memories WHERE id = ?', [id]);
+      }
+    });
+  }
+
+  return { removed: toRemove.size, groups };
+}
+
+export function memoryTopics(
+  provider: PersistenceProvider,
+): Array<{ topic: string; count: number }> {
+  const rows = provider.all<{ topics: string }>(
+    'SELECT topics FROM memories WHERE archived_at IS NULL',
+  );
+
+  const topicCounts = new Map<string, number>();
+  for (const row of rows) {
+    const topics: string[] = JSON.parse(row.topics || '[]');
+    for (const topic of topics) {
+      topicCounts.set(topic, (topicCounts.get(topic) ?? 0) + 1);
+    }
+  }
+
+  return [...topicCounts.entries()]
+    .map(([topic, count]) => ({ topic, count }))
+    .sort((a, b) => b.count - a.count);
+}
+
+export function memoriesByProject(
+  provider: PersistenceProvider,
+): Array<{ project: string; count: number; memories: Memory[] }> {
+  const rows = provider.all<{ project: string; count: number }>(
+    'SELECT project_path as project, COUNT(*) as count FROM memories WHERE archived_at IS NULL GROUP BY project_path ORDER BY count DESC',
+  );
+
+  return rows.map((row) => {
+    const mems = provider.all<Record<string, unknown>>(
+      'SELECT * FROM memories WHERE project_path = ? AND archived_at IS NULL ORDER BY created_at DESC',
+      [row.project],
+    );
+    return {
+      project: row.project,
+      count: row.count,
+      memories: mems.map(rowToMemory),
+    };
+  });
+}
+
+// ── Helper ──────────────────────────────────────────────────────────────
+
+export function rowToMemory(row: Record<string, unknown>): Memory {
+  return {
+    id: row.id as string,
+    projectPath: row.project_path as string,
+    type: row.type as Memory['type'],
+    context: row.context as string,
+    summary: row.summary as string,
+    topics: JSON.parse((row.topics as string) || '[]'),
+    filesModified: JSON.parse((row.files_modified as string) || '[]'),
+    toolsUsed: JSON.parse((row.tools_used as string) || '[]'),
+    intent: (row.intent as string) ?? null,
+    decisions: JSON.parse((row.decisions as string) || '[]'),
+    currentState: (row.current_state as string) ?? null,
+    nextSteps: JSON.parse((row.next_steps as string) || '[]'),
+    vaultEntriesReferenced: JSON.parse((row.vault_entries_referenced as string) || '[]'),
+    createdAt: row.created_at as number,
+    archivedAt: (row.archived_at as number) ?? null,
+  };
+}

package/src/{__tests__ → vault}/vault-scaling.test.ts

@@ -6,7 +6,7 @@
  */
 
 import { describe, test, expect, afterEach } from 'vitest';
-import { Vault } from '../vault/vault.js';
+import { Vault } from './vault.js';
 import { Brain } from '../brain/brain.js';
 import type { IntelligenceEntry } from '../intelligence/types.js';